PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     citype => {
 544         optional => 1,
 545         type => 'string',
 546         description => 'Specifies the cloud-init configuration format.',
 547         enum => ['configdrive2', 'nocloud'],
 548     },
 549     ciuser => {
 550         optional => 1,
 551         type => 'string',
 552         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 553     },
 554     cipassword => {
 555         optional => 1,
 556         type => 'string',
 557         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. '
 558                      . 'Also note that older cloud-init versions do not support hashed passwords.',
 559     },
 560     searchdomain => {
 561         optional => 1,
 562         type => 'string',
 563         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 564     },
 565     nameserver => {
 566         optional => 1,
 567         type => 'string', format => 'address-list',
 568         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 569     },
 570     sshkeys => {
 571         optional => 1,
 572         type => 'string',
 573         format => 'urlencoded',
 574         description => "cloud-init : Setup public SSH keys (one key per line, " .
 575                         "OpenSSH format).",
 576     },
 577 };
 578
 579 # what about other qemu settings ?
 580 #cpu => 'string',
 581 #machine => 'string',
 582 #fda => 'file',
 583 #fdb => 'file',
 584 #mtdblock => 'file',
 585 #sd => 'file',
 586 #pflash => 'file',
 587 #snapshot => 'bool',
 588 #bootp => 'file',
 589 ##tftp => 'dir',
 590 ##smb => 'dir',
 591 #kernel => 'file',
 592 #append => 'string',
 593 #initrd => 'file',
 594 ##soundhw => 'string',
 595
 596 while (my ($k, $v) = each %$confdesc) {
 597     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 598 }
 599
 600 my $MAX_IDE_DISKS = 4;
 601 my $MAX_SCSI_DISKS = 14;
 602 my $MAX_VIRTIO_DISKS = 16;
 603 my $MAX_SATA_DISKS = 6;
 604 my $MAX_USB_DEVICES = 5;
 605 my $MAX_NETS = 32;
 606 my $MAX_UNUSED_DISKS = 8;
 607 my $MAX_HOSTPCI_DEVICES = 4;
 608 my $MAX_SERIAL_PORTS = 4;
 609 my $MAX_PARALLEL_PORTS = 3;
 610 my $MAX_NUMA = 8;
 611
 612 my $numa_fmt = {
 613     cpus => {
 614         type => "string",
 615         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 616         description => "CPUs accessing this NUMA node.",
 617         format_description => "id[-id];...",
 618     },
 619     memory => {
 620         type => "number",
 621         description => "Amount of memory this NUMA node provides.",
 622         optional => 1,
 623     },
 624     hostnodes => {
 625         type => "string",
 626         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 627         description => "Host NUMA nodes to use.",
 628         format_description => "id[-id];...",
 629         optional => 1,
 630     },
 631     policy => {
 632         type => 'string',
 633         enum => [qw(preferred bind interleave)],
 634         description => "NUMA allocation policy.",
 635         optional => 1,
 636     },
 637 };
 638 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 639 my $numadesc = {
 640     optional => 1,
 641     type => 'string', format => $numa_fmt,
 642     description => "NUMA topology.",
 643 };
 644 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 645
 646 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 647     $confdesc->{"numa$i"} = $numadesc;
 648 }
 649
 650 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 651                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 652                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 653 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 654
 655 my $net_fmt_bridge_descr = <<__EOD__;
 656 Bridge to attach the network device to. The Proxmox VE standard bridge
 657 is called 'vmbr0'.
 658
 659 If you do not specify a bridge, we create a kvm user (NATed) network
 660 device, which provides DHCP and DNS services. The following addresses
 661 are used:
 662
 663  10.0.2.2   Gateway
 664  10.0.2.3   DNS Server
 665  10.0.2.4   SMB Server
 666
 667 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 668 __EOD__
 669
 670 my $net_fmt = {
 671     macaddr => {
 672         type => 'string',
 673         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 674         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 675         format_description => "XX:XX:XX:XX:XX:XX",
 676         optional => 1,
 677     },
 678     model => {
 679         type => 'string',
 680         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 681         enum => $nic_model_list,
 682         default_key => 1,
 683     },
 684     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 685     bridge => {
 686         type => 'string',
 687         description => $net_fmt_bridge_descr,
 688         format_description => 'bridge',
 689         optional => 1,
 690     },
 691     queues => {
 692         type => 'integer',
 693         minimum => 0, maximum => 16,
 694         description => 'Number of packet queues to be used on the device.',
 695         optional => 1,
 696     },
 697     rate => {
 698         type => 'number',
 699         minimum => 0,
 700         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 701         optional => 1,
 702     },
 703     tag => {
 704         type => 'integer',
 705         minimum => 1, maximum => 4094,
 706         description => 'VLAN tag to apply to packets on this interface.',
 707         optional => 1,
 708     },
 709     trunks => {
 710         type => 'string',
 711         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 712         description => 'VLAN trunks to pass through this interface.',
 713         format_description => 'vlanid[;vlanid...]',
 714         optional => 1,
 715     },
 716     firewall => {
 717         type => 'boolean',
 718         description => 'Whether this interface should be protected by the firewall.',
 719         optional => 1,
 720     },
 721     link_down => {
 722         type => 'boolean',
 723         description => 'Whether this interface should be disconnected (like pulling the plug).',
 724         optional => 1,
 725     },
 726 };
 727
 728 my $netdesc = {
 729     optional => 1,
 730     type => 'string', format => $net_fmt,
 731     description => "Specify network devices.",
 732 };
 733
 734 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 735
 736 my $ipconfig_fmt = {
 737     ip => {
 738         type => 'string',
 739         format => 'pve-ipv4-config',
 740         format_description => 'IPv4Format/CIDR',
 741         description => 'IPv4 address in CIDR format.',
 742         optional => 1,
 743         default => 'dhcp',
 744     },
 745     gw => {
 746         type => 'string',
 747         format => 'ipv4',
 748         format_description => 'GatewayIPv4',
 749         description => 'Default gateway for IPv4 traffic.',
 750         optional => 1,
 751         requires => 'ip',
 752     },
 753     ip6 => {
 754         type => 'string',
 755         format => 'pve-ipv6-config',
 756         format_description => 'IPv6Format/CIDR',
 757         description => 'IPv6 address in CIDR format.',
 758         optional => 1,
 759         default => 'dhcp',
 760     },
 761     gw6 => {
 762         type => 'string',
 763         format => 'ipv6',
 764         format_description => 'GatewayIPv6',
 765         description => 'Default gateway for IPv6 traffic.',
 766         optional => 1,
 767         requires => 'ip6',
 768     },
 769 };
 770 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 771 my $ipconfigdesc = {
 772     optional => 1,
 773     type => 'string', format => 'pve-qm-ipconfig',
 774     description => <<'EODESCR',
 775 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 776
 777 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 778
 779 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 780 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 781
 782 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 783 EODESCR
 784 };
 785 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 786
 787 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 788     $confdesc->{"net$i"} = $netdesc;
 789     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 790 }
 791
 792 foreach my $key (keys %$confdesc_cloudinit) {
 793     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 794 }
 795
 796 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 797 sub verify_volume_id_or_qm_path {
 798     my ($volid, $noerr) = @_;
 799
 800     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 801         return $volid;
 802     }
 803
 804     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 805     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 806     if ($@) {
 807         return undef if $noerr;
 808         die $@;
 809     }
 810     return $volid;
 811 }
 812
 813 my $drivename_hash;
 814
 815 my %drivedesc_base = (
 816     volume => { alias => 'file' },
 817     file => {
 818         type => 'string',
 819         format => 'pve-volume-id-or-qm-path',
 820         default_key => 1,
 821         format_description => 'volume',
 822         description => "The drive's backing volume.",
 823     },
 824     media => {
 825         type => 'string',
 826         enum => [qw(cdrom disk)],
 827         description => "The drive's media type.",
 828         default => 'disk',
 829         optional => 1
 830     },
 831     cyls => {
 832         type => 'integer',
 833         description => "Force the drive's physical geometry to have a specific cylinder count.",
 834         optional => 1
 835     },
 836     heads => {
 837         type => 'integer',
 838         description => "Force the drive's physical geometry to have a specific head count.",
 839         optional => 1
 840     },
 841     secs => {
 842         type => 'integer',
 843         description => "Force the drive's physical geometry to have a specific sector count.",
 844         optional => 1
 845     },
 846     trans => {
 847         type => 'string',
 848         enum => [qw(none lba auto)],
 849         description => "Force disk geometry bios translation mode.",
 850         optional => 1,
 851     },
 852     snapshot => {
 853         type => 'boolean',
 854         description => "Controls qemu's snapshot mode feature."
 855             . " If activated, changes made to the disk are temporary and will"
 856             . " be discarded when the VM is shutdown.",
 857         optional => 1,
 858     },
 859     cache => {
 860         type => 'string',
 861         enum => [qw(none writethrough writeback unsafe directsync)],
 862         description => "The drive's cache mode",
 863         optional => 1,
 864     },
 865     format => get_standard_option('pve-qm-image-format'),
 866     size => {
 867         type => 'string',
 868         format => 'disk-size',
 869         format_description => 'DiskSize',
 870         description => "Disk size. This is purely informational and has no effect.",
 871         optional => 1,
 872     },
 873     backup => {
 874         type => 'boolean',
 875         description => "Whether the drive should be included when making backups.",
 876         optional => 1,
 877     },
 878     replicate => {
 879         type => 'boolean',
 880         description => 'Whether the drive should considered for replication jobs.',
 881         optional => 1,
 882         default => 1,
 883     },
 884     rerror => {
 885         type => 'string',
 886         enum => [qw(ignore report stop)],
 887         description => 'Read error action.',
 888         optional => 1,
 889     },
 890     werror => {
 891         type => 'string',
 892         enum => [qw(enospc ignore report stop)],
 893         description => 'Write error action.',
 894         optional => 1,
 895     },
 896     aio => {
 897         type => 'string',
 898         enum => [qw(native threads)],
 899         description => 'AIO type to use.',
 900         optional => 1,
 901     },
 902     discard => {
 903         type => 'string',
 904         enum => [qw(ignore on)],
 905         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 906         optional => 1,
 907     },
 908     detect_zeroes => {
 909         type => 'boolean',
 910         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 911         optional => 1,
 912     },
 913     serial => {
 914         type => 'string',
 915         format => 'urlencoded',
 916         format_description => 'serial',
 917         maxLength => 20*3, # *3 since it's %xx url enoded
 918         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 919         optional => 1,
 920     },
 921     shared => {
 922         type => 'boolean',
 923         description => 'Mark this locally-managed volume as available on all nodes',
 924         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 925         optional => 1,
 926         default => 0,
 927     }
 928 );
 929
 930 my %iothread_fmt = ( iothread => {
 931         type => 'boolean',
 932         description => "Whether to use iothreads for this drive",
 933         optional => 1,
 934 });
 935
 936 my %model_fmt = (
 937     model => {
 938         type => 'string',
 939         format => 'urlencoded',
 940         format_description => 'model',
 941         maxLength => 40*3, # *3 since it's %xx url enoded
 942         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 943         optional => 1,
 944     },
 945 );
 946
 947 my %queues_fmt = (
 948     queues => {
 949         type => 'integer',
 950         description => "Number of queues.",
 951         minimum => 2,
 952         optional => 1
 953     }
 954 );
 955
 956 my %scsiblock_fmt = (
 957     scsiblock => {
 958         type => 'boolean',
 959         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 960         optional => 1,
 961         default => 0,
 962     },
 963 );
 964
 965 my $add_throttle_desc = sub {
 966     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 967     my $d = {
 968         type => $type,
 969         format_description => $unit,
 970         description => "Maximum $what in $longunit.",
 971         optional => 1,
 972     };
 973     $d->{minimum} = $minimum if defined($minimum);
 974     $drivedesc_base{$key} = $d;
 975 };
 976 # throughput: (leaky bucket)
 977 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 979 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 980 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 982 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 983 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 984 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 985 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 986
 987 # pools: (pool of IO before throttling starts taking effect)
 988 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 990 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 991 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 992 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 993 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 994
 995 # burst lengths
 996 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 998 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1001 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1002
1003 # legacy support
1004 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1005 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1006 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1007 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1008
1009 my $ide_fmt = {
1010     %drivedesc_base,
1011     %model_fmt,
1012 };
1013 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1014
1015 my $idedesc = {
1016     optional => 1,
1017     type => 'string', format => $ide_fmt,
1018     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1019 };
1020 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1021
1022 my $scsi_fmt = {
1023     %drivedesc_base,
1024     %iothread_fmt,
1025     %queues_fmt,
1026     %scsiblock_fmt,
1027 };
1028 my $scsidesc = {
1029     optional => 1,
1030     type => 'string', format => $scsi_fmt,
1031     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1032 };
1033 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1034
1035 my $sata_fmt = {
1036     %drivedesc_base,
1037 };
1038 my $satadesc = {
1039     optional => 1,
1040     type => 'string', format => $sata_fmt,
1041     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1044
1045 my $virtio_fmt = {
1046     %drivedesc_base,
1047     %iothread_fmt,
1048 };
1049 my $virtiodesc = {
1050     optional => 1,
1051     type => 'string', format => $virtio_fmt,
1052     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1053 };
1054 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1055
1056 my $alldrive_fmt = {
1057     %drivedesc_base,
1058     %iothread_fmt,
1059     %model_fmt,
1060     %queues_fmt,
1061     %scsiblock_fmt,
1062 };
1063
1064 my $efidisk_fmt = {
1065     volume => { alias => 'file' },
1066     file => {
1067         type => 'string',
1068         format => 'pve-volume-id-or-qm-path',
1069         default_key => 1,
1070         format_description => 'volume',
1071         description => "The drive's backing volume.",
1072     },
1073     format => get_standard_option('pve-qm-image-format'),
1074     size => {
1075         type => 'string',
1076         format => 'disk-size',
1077         format_description => 'DiskSize',
1078         description => "Disk size. This is purely informational and has no effect.",
1079         optional => 1,
1080     },
1081 };
1082
1083 my $efidisk_desc = {
1084     optional => 1,
1085     type => 'string', format => $efidisk_fmt,
1086     description => "Configure a Disk for storing EFI vars",
1087 };
1088
1089 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1090
1091 my $usb_fmt = {
1092     host => {
1093         default_key => 1,
1094         type => 'string', format => 'pve-qm-usb-device',
1095         format_description => 'HOSTUSBDEVICE|spice',
1096         description => <<EODESCR,
1097 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1098
1099  'bus-port(.port)*' (decimal numbers) or
1100  'vendor_id:product_id' (hexadeciaml numbers) or
1101  'spice'
1102
1103 You can use the 'lsusb -t' command to list existing usb devices.
1104
1105 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1106
1107 The value 'spice' can be used to add a usb redirection devices for spice.
1108 EODESCR
1109     },
1110     usb3 => {
1111         optional => 1,
1112         type => 'boolean',
1113         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1114         default => 0,
1115     },
1116 };
1117
1118 my $usbdesc = {
1119     optional => 1,
1120     type => 'string', format => $usb_fmt,
1121     description => "Configure an USB device (n is 0 to 4).",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1124
1125 # NOTE: the match-groups of this regex are used in parse_hostpci
1126 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1127 my $hostpci_fmt = {
1128     host => {
1129         default_key => 1,
1130         type => 'string',
1131         pattern => qr/$PCIRE(;$PCIRE)*/,
1132         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1133         description => <<EODESCR,
1134 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1135 of PCI virtual functions of the host. HOSTPCIID syntax is:
1136
1137 'bus:dev.func' (hexadecimal numbers)
1138
1139 You can us the 'lspci' command to list existing PCI devices.
1140 EODESCR
1141     },
1142     rombar => {
1143         type => 'boolean',
1144         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1145         optional => 1,
1146         default => 1,
1147     },
1148     romfile => {
1149         type => 'string',
1150         pattern => '[^,;]+',
1151         format_description => 'string',
1152         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1153         optional => 1,
1154     },
1155     pcie => {
1156         type => 'boolean',
1157         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1158         optional => 1,
1159         default => 0,
1160     },
1161     'x-vga' => {
1162         type => 'boolean',
1163         description =>  "Enable vfio-vga device support.",
1164         optional => 1,
1165         default => 0,
1166     },
1167 };
1168 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1169
1170 my $hostpcidesc = {
1171         optional => 1,
1172         type => 'string', format => 'pve-qm-hostpci',
1173         description => "Map host PCI devices into guest.",
1174         verbose_description =>  <<EODESCR,
1175 Map host PCI devices into guest.
1176
1177 NOTE: This option allows direct access to host hardware. So it is no longer
1178 possible to migrate such machines - use with special care.
1179
1180 CAUTION: Experimental! User reported problems with this option.
1181 EODESCR
1182 };
1183 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1184
1185 my $serialdesc = {
1186         optional => 1,
1187         type => 'string',
1188         pattern => '(/dev/.+|socket)',
1189         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1190         verbose_description =>  <<EODESCR,
1191 Create a serial device inside the VM (n is 0 to 3), and pass through a
1192 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1193 host side (use 'qm terminal' to open a terminal connection).
1194
1195 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1196
1197 CAUTION: Experimental! User reported problems with this option.
1198 EODESCR
1199 };
1200
1201 my $paralleldesc= {
1202         optional => 1,
1203         type => 'string',
1204         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1205         description =>  "Map host parallel devices (n is 0 to 2).",
1206         verbose_description =>  <<EODESCR,
1207 Map host parallel devices (n is 0 to 2).
1208
1209 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1210
1211 CAUTION: Experimental! User reported problems with this option.
1212 EODESCR
1213 };
1214
1215 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1216     $confdesc->{"parallel$i"} = $paralleldesc;
1217 }
1218
1219 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1220     $confdesc->{"serial$i"} = $serialdesc;
1221 }
1222
1223 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1224     $confdesc->{"hostpci$i"} = $hostpcidesc;
1225 }
1226
1227 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1228     $drivename_hash->{"ide$i"} = 1;
1229     $confdesc->{"ide$i"} = $idedesc;
1230 }
1231
1232 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1233     $drivename_hash->{"sata$i"} = 1;
1234     $confdesc->{"sata$i"} = $satadesc;
1235 }
1236
1237 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1238     $drivename_hash->{"scsi$i"} = 1;
1239     $confdesc->{"scsi$i"} = $scsidesc ;
1240 }
1241
1242 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1243     $drivename_hash->{"virtio$i"} = 1;
1244     $confdesc->{"virtio$i"} = $virtiodesc;
1245 }
1246
1247 $drivename_hash->{efidisk0} = 1;
1248 $confdesc->{efidisk0} = $efidisk_desc;
1249
1250 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1251     $confdesc->{"usb$i"} = $usbdesc;
1252 }
1253
1254 my $unuseddesc = {
1255     optional => 1,
1256     type => 'string', format => 'pve-volume-id',
1257     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1258 };
1259
1260 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1261     $confdesc->{"unused$i"} = $unuseddesc;
1262 }
1263
1264 my $kvm_api_version = 0;
1265
1266 sub kvm_version {
1267
1268     return $kvm_api_version if $kvm_api_version;
1269
1270     my $fh = IO::File->new("</dev/kvm") ||
1271         return 0;
1272
1273     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1274         $kvm_api_version = $v;
1275     }
1276
1277     $fh->close();
1278
1279     return  $kvm_api_version;
1280 }
1281
1282 my $kvm_user_version;
1283
1284 sub kvm_user_version {
1285
1286     return $kvm_user_version if $kvm_user_version;
1287
1288     $kvm_user_version = 'unknown';
1289
1290     my $code = sub {
1291         my $line = shift;
1292         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1293             $kvm_user_version = $2;
1294         }
1295     };
1296
1297     eval { run_command("kvm -version", outfunc => $code); };
1298     warn $@ if $@;
1299
1300     return $kvm_user_version;
1301
1302 }
1303
1304 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1305
1306 sub valid_drive_names {
1307     # order is important - used to autoselect boot disk
1308     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1309             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1310             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1311             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1312             'efidisk0');
1313 }
1314
1315 sub is_valid_drivename {
1316     my $dev = shift;
1317
1318     return defined($drivename_hash->{$dev});
1319 }
1320
1321 sub option_exists {
1322     my $key = shift;
1323     return defined($confdesc->{$key});
1324 }
1325
1326 sub nic_models {
1327     return $nic_model_list;
1328 }
1329
1330 sub os_list_description {
1331
1332     return {
1333         other => 'Other',
1334         wxp => 'Windows XP',
1335         w2k => 'Windows 2000',
1336         w2k3 =>, 'Windows 2003',
1337         w2k8 => 'Windows 2008',
1338         wvista => 'Windows Vista',
1339         win7 => 'Windows 7',
1340         win8 => 'Windows 8/2012',
1341         win10 => 'Windows 10/2016',
1342         l24 => 'Linux 2.4',
1343         l26 => 'Linux 2.6',
1344     };
1345 }
1346
1347 my $cdrom_path;
1348
1349 sub get_cdrom_path {
1350
1351     return  $cdrom_path if $cdrom_path;
1352
1353     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1354     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1355     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1356 }
1357
1358 sub get_iso_path {
1359     my ($storecfg, $vmid, $cdrom) = @_;
1360
1361     if ($cdrom eq 'cdrom') {
1362         return get_cdrom_path();
1363     } elsif ($cdrom eq 'none') {
1364         return '';
1365     } elsif ($cdrom =~ m|^/|) {
1366         return $cdrom;
1367     } else {
1368         return PVE::Storage::path($storecfg, $cdrom);
1369     }
1370 }
1371
1372 # try to convert old style file names to volume IDs
1373 sub filename_to_volume_id {
1374     my ($vmid, $file, $media) = @_;
1375
1376      if (!($file eq 'none' || $file eq 'cdrom' ||
1377           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1378
1379         return undef if $file =~ m|/|;
1380
1381         if ($media && $media eq 'cdrom') {
1382             $file = "local:iso/$file";
1383         } else {
1384             $file = "local:$vmid/$file";
1385         }
1386     }
1387
1388     return $file;
1389 }
1390
1391 sub verify_media_type {
1392     my ($opt, $vtype, $media) = @_;
1393
1394     return if !$media;
1395
1396     my $etype;
1397     if ($media eq 'disk') {
1398         $etype = 'images';
1399     } elsif ($media eq 'cdrom') {
1400         $etype = 'iso';
1401     } else {
1402         die "internal error";
1403     }
1404
1405     return if ($vtype eq $etype);
1406
1407     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1408 }
1409
1410 sub cleanup_drive_path {
1411     my ($opt, $storecfg, $drive) = @_;
1412
1413     # try to convert filesystem paths to volume IDs
1414
1415     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1416         ($drive->{file} !~ m|^/dev/.+|) &&
1417         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1418         ($drive->{file} !~ m/^\d+$/)) {
1419         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1420         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1421         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1422         verify_media_type($opt, $vtype, $drive->{media});
1423         $drive->{file} = $volid;
1424     }
1425
1426     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1427 }
1428
1429 sub parse_hotplug_features {
1430     my ($data) = @_;
1431
1432     my $res = {};
1433
1434     return $res if $data eq '0';
1435
1436     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1437
1438     foreach my $feature (PVE::Tools::split_list($data)) {
1439         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1440             $res->{$1} = 1;
1441         } else {
1442             die "invalid hotplug feature '$feature'\n";
1443         }
1444     }
1445     return $res;
1446 }
1447
1448 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1449 sub pve_verify_hotplug_features {
1450     my ($value, $noerr) = @_;
1451
1452     return $value if parse_hotplug_features($value);
1453
1454     return undef if $noerr;
1455
1456     die "unable to parse hotplug option\n";
1457 }
1458
1459 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1460 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1461 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1462 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1463 #        [,iothread=on][,serial=serial][,model=model]
1464
1465 sub parse_drive {
1466     my ($key, $data) = @_;
1467
1468     my ($interface, $index);
1469
1470     if ($key =~ m/^([^\d]+)(\d+)$/) {
1471         $interface = $1;
1472         $index = $2;
1473     } else {
1474         return undef;
1475     }
1476
1477     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1478                                      : $confdesc->{$key}->{format};
1479     if (!$desc) {
1480         warn "invalid drive key: $key\n";
1481         return undef;
1482     }
1483     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1484     return undef if !$res;
1485     $res->{interface} = $interface;
1486     $res->{index} = $index;
1487
1488     my $error = 0;
1489     foreach my $opt (qw(bps bps_rd bps_wr)) {
1490         if (my $bps = defined(delete $res->{$opt})) {
1491             if (defined($res->{"m$opt"})) {
1492                 warn "both $opt and m$opt specified\n";
1493                 ++$error;
1494                 next;
1495             }
1496             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1497         }
1498     }
1499
1500     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1501     for my $requirement (
1502         [mbps_max => 'mbps'],
1503         [mbps_rd_max => 'mbps_rd'],
1504         [mbps_wr_max => 'mbps_wr'],
1505         [miops_max => 'miops'],
1506         [miops_rd_max => 'miops_rd'],
1507         [miops_wr_max => 'miops_wr'],
1508         [bps_max_length => 'mbps_max'],
1509         [bps_rd_max_length => 'mbps_rd_max'],
1510         [bps_wr_max_length => 'mbps_wr_max'],
1511         [iops_max_length => 'iops_max'],
1512         [iops_rd_max_length => 'iops_rd_max'],
1513         [iops_wr_max_length => 'iops_wr_max']) {
1514         my ($option, $requires) = @$requirement;
1515         if ($res->{$option} && !$res->{$requires}) {
1516             warn "$option requires $requires\n";
1517             ++$error;
1518         }
1519     }
1520
1521     return undef if $error;
1522
1523     return undef if $res->{mbps_rd} && $res->{mbps};
1524     return undef if $res->{mbps_wr} && $res->{mbps};
1525     return undef if $res->{iops_rd} && $res->{iops};
1526     return undef if $res->{iops_wr} && $res->{iops};
1527
1528     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1529         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1530         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1531         return undef if $res->{interface} eq 'virtio';
1532     }
1533
1534     if (my $size = $res->{size}) {
1535         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1536     }
1537
1538     return $res;
1539 }
1540
1541 sub print_drive {
1542     my ($vmid, $drive) = @_;
1543     my $data = { %$drive };
1544     delete $data->{$_} for qw(index interface);
1545     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1546 }
1547
1548 sub scsi_inquiry {
1549     my($fh, $noerr) = @_;
1550
1551     my $SG_IO = 0x2285;
1552     my $SG_GET_VERSION_NUM = 0x2282;
1553
1554     my $versionbuf = "\x00" x 8;
1555     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1556     if (!$ret) {
1557         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1558         return undef;
1559     }
1560     my $version = unpack("I", $versionbuf);
1561     if ($version < 30000) {
1562         die "scsi generic interface too old\n"  if !$noerr;
1563         return undef;
1564     }
1565
1566     my $buf = "\x00" x 36;
1567     my $sensebuf = "\x00" x 8;
1568     my $cmd = pack("C x3 C x1", 0x12, 36);
1569
1570     # see /usr/include/scsi/sg.h
1571     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1572
1573     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1574                       length($sensebuf), 0, length($buf), $buf,
1575                       $cmd, $sensebuf, 6000);
1576
1577     $ret = ioctl($fh, $SG_IO, $packet);
1578     if (!$ret) {
1579         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1580         return undef;
1581     }
1582
1583     my @res = unpack($sg_io_hdr_t, $packet);
1584     if ($res[17] || $res[18]) {
1585         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1586         return undef;
1587     }
1588
1589     my $res = {};
1590     (my $byte0, my $byte1, $res->{vendor},
1591      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1592
1593     $res->{removable} = $byte1 & 128 ? 1 : 0;
1594     $res->{type} = $byte0 & 31;
1595
1596     return $res;
1597 }
1598
1599 sub path_is_scsi {
1600     my ($path) = @_;
1601
1602     my $fh = IO::File->new("+<$path") || return undef;
1603     my $res = scsi_inquiry($fh, 1);
1604     close($fh);
1605
1606     return $res;
1607 }
1608
1609 sub machine_type_is_q35 {
1610     my ($conf) = @_;
1611
1612     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1613 }
1614
1615 sub print_tabletdevice_full {
1616     my ($conf) = @_;
1617
1618     my $q35 = machine_type_is_q35($conf);
1619
1620     # we use uhci for old VMs because tablet driver was buggy in older qemu
1621     my $usbbus = $q35 ? "ehci" : "uhci";
1622
1623     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1624 }
1625
1626 sub print_drivedevice_full {
1627     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1628
1629     my $device = '';
1630     my $maxdev = 0;
1631
1632     if ($drive->{interface} eq 'virtio') {
1633         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1634         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1635         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1636     } elsif ($drive->{interface} eq 'scsi') {
1637
1638         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1639         my $unit = $drive->{index} % $maxdev;
1640         my $devicetype = 'hd';
1641         my $path = '';
1642         if (drive_is_cdrom($drive)) {
1643             $devicetype = 'cd';
1644         } else {
1645             if ($drive->{file} =~ m|^/|) {
1646                 $path = $drive->{file};
1647                 if (my $info = path_is_scsi($path)) {
1648                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1649                         $devicetype = 'block';
1650                     } elsif ($info->{type} == 1) { # tape
1651                         $devicetype = 'generic';
1652                     }
1653                 }
1654             } else {
1655                  $path = PVE::Storage::path($storecfg, $drive->{file});
1656             }
1657
1658             if($path =~ m/^iscsi\:\/\//){
1659                 $devicetype = 'generic';
1660             }
1661         }
1662
1663         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         } else {
1666             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1667         }
1668
1669     } elsif ($drive->{interface} eq 'ide'){
1670         $maxdev = 2;
1671         my $controller = int($drive->{index} / $maxdev);
1672         my $unit = $drive->{index} % $maxdev;
1673         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1674
1675         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1676         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1677             $model = URI::Escape::uri_unescape($model);
1678             $device .= ",model=$model";
1679         }
1680     } elsif ($drive->{interface} eq 'sata'){
1681         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1682         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1683         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1684     } elsif ($drive->{interface} eq 'usb') {
1685         die "implement me";
1686         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1687     } else {
1688         die "unsupported interface type";
1689     }
1690
1691     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1692
1693     return $device;
1694 }
1695
1696 sub get_initiator_name {
1697     my $initiator;
1698
1699     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1700     while (defined(my $line = <$fh>)) {
1701         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1702         $initiator = $1;
1703         last;
1704     }
1705     $fh->close();
1706
1707     return $initiator;
1708 }
1709
1710 sub print_drive_full {
1711     my ($storecfg, $vmid, $drive) = @_;
1712
1713     my $path;
1714     my $volid = $drive->{file};
1715     my $format;
1716
1717     if (drive_is_cdrom($drive)) {
1718         $path = get_iso_path($storecfg, $vmid, $volid);
1719     } else {
1720         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1721         if ($storeid) {
1722             $path = PVE::Storage::path($storecfg, $volid);
1723             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1724             $format = qemu_img_format($scfg, $volname);
1725         } else {
1726             $path = $volid;
1727             $format = "raw";
1728         }
1729    }
1730
1731     my $opts = '';
1732     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1733     foreach my $o (@qemu_drive_options) {
1734         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1735     }
1736
1737     # snapshot only accepts on|off
1738     if (defined($drive->{snapshot})) {
1739         my $v = $drive->{snapshot} ? 'on' : 'off';
1740         $opts .= ",snapshot=$v";
1741     }
1742
1743     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1744         my ($dir, $qmpname) = @$type;
1745         if (my $v = $drive->{"mbps$dir"}) {
1746             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1747         }
1748         if (my $v = $drive->{"mbps${dir}_max"}) {
1749             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1750         }
1751         if (my $v = $drive->{"bps${dir}_max_length"}) {
1752             $opts .= ",throttling.bps$qmpname-max-length=$v";
1753         }
1754         if (my $v = $drive->{"iops${dir}"}) {
1755             $opts .= ",throttling.iops$qmpname=$v";
1756         }
1757         if (my $v = $drive->{"iops${dir}_max"}) {
1758             $opts .= ",throttling.iops$qmpname-max=$v";
1759         }
1760         if (my $v = $drive->{"iops${dir}_max_length"}) {
1761             $opts .= ",throttling.iops$qmpname-max-length=$v";
1762         }
1763     }
1764
1765     if (my $serial = $drive->{serial}) {
1766         $serial = URI::Escape::uri_unescape($serial);
1767         $opts .= ",serial=$serial";
1768     }
1769
1770     $opts .= ",format=$format" if $format && !$drive->{format};
1771
1772     my $cache_direct = 0;
1773
1774     if (my $cache = $drive->{cache}) {
1775         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1776     } elsif (!drive_is_cdrom($drive)) {
1777         $opts .= ",cache=none";
1778         $cache_direct = 1;
1779     }
1780
1781     # aio native works only with O_DIRECT
1782     if (!$drive->{aio}) {
1783         if($cache_direct) {
1784             $opts .= ",aio=native";
1785         } else {
1786             $opts .= ",aio=threads";
1787         }
1788     }
1789
1790     if (!drive_is_cdrom($drive)) {
1791         my $detectzeroes;
1792         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1793             $detectzeroes = 'off';
1794         } elsif ($drive->{discard}) {
1795             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1796         } else {
1797             # This used to be our default with discard not being specified:
1798             $detectzeroes = 'on';
1799         }
1800         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1801     }
1802
1803     my $pathinfo = $path ? "file=$path," : '';
1804
1805     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1806 }
1807
1808 sub print_netdevice_full {
1809     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1810
1811     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1812
1813     my $device = $net->{model};
1814     if ($net->{model} eq 'virtio') {
1815          $device = 'virtio-net-pci';
1816      };
1817
1818     my $pciaddr = print_pci_addr("$netid", $bridges);
1819     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1820     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1821         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1822         my $vectors = $net->{queues} * 2 + 2;
1823         $tmpstr .= ",vectors=$vectors,mq=on";
1824     }
1825     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1826
1827     if ($use_old_bios_files) {
1828         my $romfile;
1829         if ($device eq 'virtio-net-pci') {
1830             $romfile = 'pxe-virtio.rom';
1831         } elsif ($device eq 'e1000') {
1832             $romfile = 'pxe-e1000.rom';
1833         } elsif ($device eq 'ne2k') {
1834             $romfile = 'pxe-ne2k_pci.rom';
1835         } elsif ($device eq 'pcnet') {
1836             $romfile = 'pxe-pcnet.rom';
1837         } elsif ($device eq 'rtl8139') {
1838             $romfile = 'pxe-rtl8139.rom';
1839         }
1840         $tmpstr .= ",romfile=$romfile" if $romfile;
1841     }
1842
1843     return $tmpstr;
1844 }
1845
1846 sub print_netdev_full {
1847     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1848
1849     my $i = '';
1850     if ($netid =~ m/^net(\d+)$/) {
1851         $i = int($1);
1852     }
1853
1854     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1855
1856     my $ifname = "tap${vmid}i$i";
1857
1858     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1859     die "interface name '$ifname' is too long (max 15 character)\n"
1860         if length($ifname) >= 16;
1861
1862     my $vhostparam = '';
1863     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1864
1865     my $vmname = $conf->{name} || "vm$vmid";
1866
1867     my $netdev = "";
1868     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1869
1870     if ($net->{bridge}) {
1871         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1872     } else {
1873         $netdev = "type=user,id=$netid,hostname=$vmname";
1874     }
1875
1876     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1877
1878     return $netdev;
1879 }
1880
1881
1882 sub print_cpu_device {
1883     my ($conf, $id) = @_;
1884
1885     my $kvm = $conf->{kvm} // 1;
1886     my $cpu = $kvm ? "kvm64" : "qemu64";
1887     if (my $cputype = $conf->{cpu}) {
1888         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1889             or die "Cannot parse cpu description: $cputype\n";
1890         $cpu = $cpuconf->{cputype};
1891     }
1892
1893     my $cores = $conf->{cores} || 1;
1894
1895     my $current_core = ($id - 1) % $cores;
1896     my $current_socket = int(($id - 1 - $current_core)/$cores);
1897
1898     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1899 }
1900
1901 sub drive_is_cloudinit {
1902     my ($drive) = @_;
1903     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1904 }
1905
1906 sub drive_is_cdrom {
1907     my ($drive, $exclude_cloudinit) = @_;
1908
1909     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1910
1911     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1912
1913 }
1914
1915 sub parse_number_sets {
1916     my ($set) = @_;
1917     my $res = [];
1918     foreach my $part (split(/;/, $set)) {
1919         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1920             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1921             push @$res, [ $1, $2 ];
1922         } else {
1923             die "invalid range: $part\n";
1924         }
1925     }
1926     return $res;
1927 }
1928
1929 sub parse_numa {
1930     my ($data) = @_;
1931
1932     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1933     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1934     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1935     return $res;
1936 }
1937
1938 sub parse_hostpci {
1939     my ($value) = @_;
1940
1941     return undef if !$value;
1942
1943     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1944
1945     my @idlist = split(/;/, $res->{host});
1946     delete $res->{host};
1947     foreach my $id (@idlist) {
1948         if ($id =~ /^$PCIRE$/) {
1949             if (defined($2)) {
1950                 push @{$res->{pciid}}, { id => $1, function => $2 };
1951             } else {
1952                 my $pcidevices = lspci($1);
1953                 $res->{pciid} = $pcidevices->{$1};
1954             }
1955         } else {
1956             # should have been caught by parse_property_string already
1957             die "failed to parse PCI id: $id\n";
1958         }
1959     }
1960     return $res;
1961 }
1962
1963 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1964 sub parse_net {
1965     my ($data) = @_;
1966
1967     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1968     if ($@) {
1969         warn $@;
1970         return undef;
1971     }
1972     if (!defined($res->{macaddr})) {
1973         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1974         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1975     }
1976     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1977     return $res;
1978 }
1979
1980 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1981 sub parse_ipconfig {
1982     my ($data) = @_;
1983
1984     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1985     if ($@) {
1986         warn $@;
1987         return undef;
1988     }
1989
1990     if ($res->{gw} && !$res->{ip}) {
1991         warn 'gateway specified without specifying an IP address';
1992         return undef;
1993     }
1994     if ($res->{gw6} && !$res->{ip6}) {
1995         warn 'IPv6 gateway specified without specifying an IPv6 address';
1996         return undef;
1997     }
1998     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1999         warn 'gateway specified together with DHCP';
2000         return undef;
2001     }
2002     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2003         # gw6 + auto/dhcp
2004         warn "IPv6 gateway specified together with $res->{ip6} address";
2005         return undef;
2006     }
2007
2008     if (!$res->{ip} && !$res->{ip6}) {
2009         return { ip => 'dhcp', ip6 => 'dhcp' };
2010     }
2011
2012     return $res;
2013 }
2014
2015 sub print_net {
2016     my $net = shift;
2017
2018     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2019 }
2020
2021 sub add_random_macs {
2022     my ($settings) = @_;
2023
2024     foreach my $opt (keys %$settings) {
2025         next if $opt !~ m/^net(\d+)$/;
2026         my $net = parse_net($settings->{$opt});
2027         next if !$net;
2028         $settings->{$opt} = print_net($net);
2029     }
2030 }
2031
2032 sub vm_is_volid_owner {
2033     my ($storecfg, $vmid, $volid) = @_;
2034
2035     if ($volid !~  m|^/|) {
2036         my ($path, $owner);
2037         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2038         if ($owner && ($owner == $vmid)) {
2039             return 1;
2040         }
2041     }
2042
2043     return undef;
2044 }
2045
2046 sub split_flagged_list {
2047     my $text = shift || '';
2048     $text =~ s/[,;]/ /g;
2049     $text =~ s/^\s+//;
2050     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2051 }
2052
2053 sub join_flagged_list {
2054     my ($how, $lst) = @_;
2055     join $how, map { $lst->{$_} . $_ } keys %$lst;
2056 }
2057
2058 sub vmconfig_delete_pending_option {
2059     my ($conf, $key, $force) = @_;
2060
2061     delete $conf->{pending}->{$key};
2062     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2063     $pending_delete_hash->{$key} = $force ? '!' : '';
2064     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2065 }
2066
2067 sub vmconfig_undelete_pending_option {
2068     my ($conf, $key) = @_;
2069
2070     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2071     delete $pending_delete_hash->{$key};
2072
2073     if (%$pending_delete_hash) {
2074         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2075     } else {
2076         delete $conf->{pending}->{delete};
2077     }
2078 }
2079
2080 sub vmconfig_register_unused_drive {
2081     my ($storecfg, $vmid, $conf, $drive) = @_;
2082
2083     if (drive_is_cloudinit($drive)) {
2084         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2085         warn $@ if $@;
2086     } elsif (!drive_is_cdrom($drive)) {
2087         my $volid = $drive->{file};
2088         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2089             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2090         }
2091     }
2092 }
2093
2094 sub vmconfig_cleanup_pending {
2095     my ($conf) = @_;
2096
2097     # remove pending changes when nothing changed
2098     my $changes;
2099     foreach my $opt (keys %{$conf->{pending}}) {
2100         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2101             $changes = 1;
2102             delete $conf->{pending}->{$opt};
2103         }
2104     }
2105
2106     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2107     my $pending_delete_hash = {};
2108     while (my ($opt, $force) = each %$current_delete_hash) {
2109         if (defined($conf->{$opt})) {
2110             $pending_delete_hash->{$opt} = $force;
2111         } else {
2112             $changes = 1;
2113         }
2114     }
2115
2116     if (%$pending_delete_hash) {
2117         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2118     } else {
2119         delete $conf->{pending}->{delete};
2120     }
2121
2122     return $changes;
2123 }
2124
2125 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2126 my $smbios1_fmt = {
2127     uuid => {
2128         type => 'string',
2129         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2130         format_description => 'UUID',
2131         description => "Set SMBIOS1 UUID.",
2132         optional => 1,
2133     },
2134     version => {
2135         type => 'string',
2136         pattern => '\S+',
2137         format_description => 'string',
2138         description => "Set SMBIOS1 version.",
2139         optional => 1,
2140     },
2141     serial => {
2142         type => 'string',
2143         pattern => '\S+',
2144         format_description => 'string',
2145         description => "Set SMBIOS1 serial number.",
2146         optional => 1,
2147     },
2148     manufacturer => {
2149         type => 'string',
2150         pattern => '\S+',
2151         format_description => 'string',
2152         description => "Set SMBIOS1 manufacturer.",
2153         optional => 1,
2154     },
2155     product => {
2156         type => 'string',
2157         pattern => '\S+',
2158         format_description => 'string',
2159         description => "Set SMBIOS1 product ID.",
2160         optional => 1,
2161     },
2162     sku => {
2163         type => 'string',
2164         pattern => '\S+',
2165         format_description => 'string',
2166         description => "Set SMBIOS1 SKU string.",
2167         optional => 1,
2168     },
2169     family => {
2170         type => 'string',
2171         pattern => '\S+',
2172         format_description => 'string',
2173         description => "Set SMBIOS1 family string.",
2174         optional => 1,
2175     },
2176 };
2177
2178 sub parse_smbios1 {
2179     my ($data) = @_;
2180
2181     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2182     warn $@ if $@;
2183     return $res;
2184 }
2185
2186 sub print_smbios1 {
2187     my ($smbios1) = @_;
2188     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2189 }
2190
2191 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2192
2193 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2194 sub verify_bootdisk {
2195     my ($value, $noerr) = @_;
2196
2197     return $value if is_valid_drivename($value);
2198
2199     return undef if $noerr;
2200
2201     die "invalid boot disk '$value'\n";
2202 }
2203
2204 sub parse_watchdog {
2205     my ($value) = @_;
2206
2207     return undef if !$value;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2210     warn $@ if $@;
2211     return $res;
2212 }
2213
2214 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2215 sub verify_usb_device {
2216     my ($value, $noerr) = @_;
2217
2218     return $value if parse_usb_device($value);
2219
2220     return undef if $noerr;
2221
2222     die "unable to parse usb device\n";
2223 }
2224
2225 # add JSON properties for create and set function
2226 sub json_config_properties {
2227     my $prop = shift;
2228
2229     foreach my $opt (keys %$confdesc) {
2230         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2231         $prop->{$opt} = $confdesc->{$opt};
2232     }
2233
2234     return $prop;
2235 }
2236
2237 sub check_type {
2238     my ($key, $value) = @_;
2239
2240     die "unknown setting '$key'\n" if !$confdesc->{$key};
2241
2242     my $type = $confdesc->{$key}->{type};
2243
2244     if (!defined($value)) {
2245         die "got undefined value\n";
2246     }
2247
2248     if ($value =~ m/[\n\r]/) {
2249         die "property contains a line feed\n";
2250     }
2251
2252     if ($type eq 'boolean') {
2253         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2254         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2255         die "type check ('boolean') failed - got '$value'\n";
2256     } elsif ($type eq 'integer') {
2257         return int($1) if $value =~ m/^(\d+)$/;
2258         die "type check ('integer') failed - got '$value'\n";
2259     } elsif ($type eq 'number') {
2260         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2261         die "type check ('number') failed - got '$value'\n";
2262     } elsif ($type eq 'string') {
2263         if (my $fmt = $confdesc->{$key}->{format}) {
2264             PVE::JSONSchema::check_format($fmt, $value);
2265             return $value;
2266         }
2267         $value =~ s/^\"(.*)\"$/$1/;
2268         return $value;
2269     } else {
2270         die "internal error"
2271     }
2272 }
2273
2274 sub check_iommu_support{
2275     #fixme : need to check IOMMU support
2276     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2277
2278     my $iommu=1;
2279     return $iommu;
2280
2281 }
2282
2283 sub touch_config {
2284     my ($vmid) = @_;
2285
2286     my $conf = PVE::QemuConfig->config_file($vmid);
2287     utime undef, undef, $conf;
2288 }
2289
2290 sub destroy_vm {
2291     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2292
2293     my $conffile = PVE::QemuConfig->config_file($vmid);
2294
2295     my $conf = PVE::QemuConfig->load_config($vmid);
2296
2297     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2298
2299     if ($conf->{template}) {
2300         # check if any base image is still used by a linked clone
2301         foreach_drive($conf, sub {
2302                 my ($ds, $drive) = @_;
2303
2304                 return if drive_is_cdrom($drive);
2305
2306                 my $volid = $drive->{file};
2307
2308                 return if !$volid || $volid =~ m|^/|;
2309
2310                 die "base volume '$volid' is still in use by linked cloned\n"
2311                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2312
2313         });
2314     }
2315
2316     # only remove disks owned by this VM
2317     foreach_drive($conf, sub {
2318         my ($ds, $drive) = @_;
2319
2320         return if drive_is_cdrom($drive, 1);
2321
2322         my $volid = $drive->{file};
2323
2324         return if !$volid || $volid =~ m|^/|;
2325
2326         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2327         return if !$path || !$owner || ($owner != $vmid);
2328
2329         eval {
2330             PVE::Storage::vdisk_free($storecfg, $volid);
2331         };
2332         warn "Could not remove disk '$volid', check manually: $@" if $@;
2333
2334     });
2335
2336     if ($keep_empty_config) {
2337         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2338     } else {
2339         unlink $conffile;
2340     }
2341
2342     # also remove unused disk
2343     eval {
2344         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2345
2346         eval {
2347             PVE::Storage::foreach_volid($dl, sub {
2348                 my ($volid, $sid, $volname, $d) = @_;
2349                 PVE::Storage::vdisk_free($storecfg, $volid);
2350             });
2351         };
2352         warn $@ if $@;
2353
2354     };
2355     warn $@ if $@;
2356 }
2357
2358 sub parse_vm_config {
2359     my ($filename, $raw) = @_;
2360
2361     return undef if !defined($raw);
2362
2363     my $res = {
2364         digest => Digest::SHA::sha1_hex($raw),
2365         snapshots => {},
2366         pending => {},
2367     };
2368
2369     $filename =~ m|/qemu-server/(\d+)\.conf$|
2370         || die "got strange filename '$filename'";
2371
2372     my $vmid = $1;
2373
2374     my $conf = $res;
2375     my $descr;
2376     my $section = '';
2377
2378     my @lines = split(/\n/, $raw);
2379     foreach my $line (@lines) {
2380         next if $line =~ m/^\s*$/;
2381
2382         if ($line =~ m/^\[PENDING\]\s*$/i) {
2383             $section = 'pending';
2384             if (defined($descr)) {
2385                 $descr =~ s/\s+$//;
2386                 $conf->{description} = $descr;
2387             }
2388             $descr = undef;
2389             $conf = $res->{$section} = {};
2390             next;
2391
2392         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2393             $section = $1;
2394             if (defined($descr)) {
2395                 $descr =~ s/\s+$//;
2396                 $conf->{description} = $descr;
2397             }
2398             $descr = undef;
2399             $conf = $res->{snapshots}->{$section} = {};
2400             next;
2401         }
2402
2403         if ($line =~ m/^\#(.*)\s*$/) {
2404             $descr = '' if !defined($descr);
2405             $descr .= PVE::Tools::decode_text($1) . "\n";
2406             next;
2407         }
2408
2409         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2410             $descr = '' if !defined($descr);
2411             $descr .= PVE::Tools::decode_text($2);
2412         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2413             $conf->{snapstate} = $1;
2414         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2415             my $key = $1;
2416             my $value = $2;
2417             $conf->{$key} = $value;
2418         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2419             my $value = $1;
2420             if ($section eq 'pending') {
2421                 $conf->{delete} = $value; # we parse this later
2422             } else {
2423                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2424             }
2425         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2426             my $key = $1;
2427             my $value = $2;
2428             eval { $value = check_type($key, $value); };
2429             if ($@) {
2430                 warn "vm $vmid - unable to parse value of '$key' - $@";
2431             } else {
2432                 $key = 'ide2' if $key eq 'cdrom';
2433                 my $fmt = $confdesc->{$key}->{format};
2434                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2435                     my $v = parse_drive($key, $value);
2436                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2437                         $v->{file} = $volid;
2438                         $value = print_drive($vmid, $v);
2439                     } else {
2440                         warn "vm $vmid - unable to parse value of '$key'\n";
2441                         next;
2442                     }
2443                 }
2444
2445                 $conf->{$key} = $value;
2446             }
2447         }
2448     }
2449
2450     if (defined($descr)) {
2451         $descr =~ s/\s+$//;
2452         $conf->{description} = $descr;
2453     }
2454     delete $res->{snapstate}; # just to be sure
2455
2456     return $res;
2457 }
2458
2459 sub write_vm_config {
2460     my ($filename, $conf) = @_;
2461
2462     delete $conf->{snapstate}; # just to be sure
2463
2464     if ($conf->{cdrom}) {
2465         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2466         $conf->{ide2} = $conf->{cdrom};
2467         delete $conf->{cdrom};
2468     }
2469
2470     # we do not use 'smp' any longer
2471     if ($conf->{sockets}) {
2472         delete $conf->{smp};
2473     } elsif ($conf->{smp}) {
2474         $conf->{sockets} = $conf->{smp};
2475         delete $conf->{cores};
2476         delete $conf->{smp};
2477     }
2478
2479     my $used_volids = {};
2480
2481     my $cleanup_config = sub {
2482         my ($cref, $pending, $snapname) = @_;
2483
2484         foreach my $key (keys %$cref) {
2485             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2486                 $key eq 'snapstate' || $key eq 'pending';
2487             my $value = $cref->{$key};
2488             if ($key eq 'delete') {
2489                 die "propertry 'delete' is only allowed in [PENDING]\n"
2490                     if !$pending;
2491                 # fixme: check syntax?
2492                 next;
2493             }
2494             eval { $value = check_type($key, $value); };
2495             die "unable to parse value of '$key' - $@" if $@;
2496
2497             $cref->{$key} = $value;
2498
2499             if (!$snapname && is_valid_drivename($key)) {
2500                 my $drive = parse_drive($key, $value);
2501                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2502             }
2503         }
2504     };
2505
2506     &$cleanup_config($conf);
2507
2508     &$cleanup_config($conf->{pending}, 1);
2509
2510     foreach my $snapname (keys %{$conf->{snapshots}}) {
2511         die "internal error" if $snapname eq 'pending';
2512         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2513     }
2514
2515     # remove 'unusedX' settings if we re-add a volume
2516     foreach my $key (keys %$conf) {
2517         my $value = $conf->{$key};
2518         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2519             delete $conf->{$key};
2520         }
2521     }
2522
2523     my $generate_raw_config = sub {
2524         my ($conf, $pending) = @_;
2525
2526         my $raw = '';
2527
2528         # add description as comment to top of file
2529         if (defined(my $descr = $conf->{description})) {
2530             if ($descr) {
2531                 foreach my $cl (split(/\n/, $descr)) {
2532                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2533                 }
2534             } else {
2535                 $raw .= "#\n" if $pending;
2536             }
2537         }
2538
2539         foreach my $key (sort keys %$conf) {
2540             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2541             $raw .= "$key: $conf->{$key}\n";
2542         }
2543         return $raw;
2544     };
2545
2546     my $raw = &$generate_raw_config($conf);
2547
2548     if (scalar(keys %{$conf->{pending}})){
2549         $raw .= "\n[PENDING]\n";
2550         $raw .= &$generate_raw_config($conf->{pending}, 1);
2551     }
2552
2553     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2554         $raw .= "\n[$snapname]\n";
2555         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2556     }
2557
2558     return $raw;
2559 }
2560
2561 sub load_defaults {
2562
2563     my $res = {};
2564
2565     # we use static defaults from our JSON schema configuration
2566     foreach my $key (keys %$confdesc) {
2567         if (defined(my $default = $confdesc->{$key}->{default})) {
2568             $res->{$key} = $default;
2569         }
2570     }
2571
2572     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2573     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2574
2575     return $res;
2576 }
2577
2578 sub config_list {
2579     my $vmlist = PVE::Cluster::get_vmlist();
2580     my $res = {};
2581     return $res if !$vmlist || !$vmlist->{ids};
2582     my $ids = $vmlist->{ids};
2583
2584     foreach my $vmid (keys %$ids) {
2585         my $d = $ids->{$vmid};
2586         next if !$d->{node} || $d->{node} ne $nodename;
2587         next if !$d->{type} || $d->{type} ne 'qemu';
2588         $res->{$vmid}->{exists} = 1;
2589     }
2590     return $res;
2591 }
2592
2593 # test if VM uses local resources (to prevent migration)
2594 sub check_local_resources {
2595     my ($conf, $noerr) = @_;
2596
2597     my $loc_res = 0;
2598
2599     $loc_res = 1 if $conf->{hostusb}; # old syntax
2600     $loc_res = 1 if $conf->{hostpci}; # old syntax
2601
2602     foreach my $k (keys %$conf) {
2603         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2604         # sockets are safe: they will recreated be on the target side post-migrate
2605         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2606         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2607     }
2608
2609     die "VM uses local resources\n" if $loc_res && !$noerr;
2610
2611     return $loc_res;
2612 }
2613
2614 # check if used storages are available on all nodes (use by migrate)
2615 sub check_storage_availability {
2616     my ($storecfg, $conf, $node) = @_;
2617
2618     foreach_drive($conf, sub {
2619         my ($ds, $drive) = @_;
2620
2621         my $volid = $drive->{file};
2622         return if !$volid;
2623
2624         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2625         return if !$sid;
2626
2627         # check if storage is available on both nodes
2628         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2629         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2630    });
2631 }
2632
2633 # list nodes where all VM images are available (used by has_feature API)
2634 sub shared_nodes {
2635     my ($conf, $storecfg) = @_;
2636
2637     my $nodelist = PVE::Cluster::get_nodelist();
2638     my $nodehash = { map { $_ => 1 } @$nodelist };
2639     my $nodename = PVE::INotify::nodename();
2640
2641     foreach_drive($conf, sub {
2642         my ($ds, $drive) = @_;
2643
2644         my $volid = $drive->{file};
2645         return if !$volid;
2646
2647         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2648         if ($storeid) {
2649             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2650             if ($scfg->{disable}) {
2651                 $nodehash = {};
2652             } elsif (my $avail = $scfg->{nodes}) {
2653                 foreach my $node (keys %$nodehash) {
2654                     delete $nodehash->{$node} if !$avail->{$node};
2655                 }
2656             } elsif (!$scfg->{shared}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     delete $nodehash->{$node} if $node ne $nodename
2659                 }
2660             }
2661         }
2662     });
2663
2664     return $nodehash
2665 }
2666
2667 sub check_cmdline {
2668     my ($pidfile, $pid) = @_;
2669
2670     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2671     if (defined($fh)) {
2672         my $line = <$fh>;
2673         $fh->close;
2674         return undef if !$line;
2675         my @param = split(/\0/, $line);
2676
2677         my $cmd = $param[0];
2678         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2679
2680         for (my $i = 0; $i < scalar (@param); $i++) {
2681             my $p = $param[$i];
2682             next if !$p;
2683             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2684                 my $p = $param[$i+1];
2685                 return 1 if $p && ($p eq $pidfile);
2686                 return undef;
2687             }
2688         }
2689     }
2690     return undef;
2691 }
2692
2693 sub check_running {
2694     my ($vmid, $nocheck, $node) = @_;
2695
2696     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2697
2698     die "unable to find configuration file for VM $vmid - no such machine\n"
2699         if !$nocheck && ! -f $filename;
2700
2701     my $pidfile = pidfile_name($vmid);
2702
2703     if (my $fd = IO::File->new("<$pidfile")) {
2704         my $st = stat($fd);
2705         my $line = <$fd>;
2706         close($fd);
2707
2708         my $mtime = $st->mtime;
2709         if ($mtime > time()) {
2710             warn "file '$filename' modified in future\n";
2711         }
2712
2713         if ($line =~ m/^(\d+)$/) {
2714             my $pid = $1;
2715             if (check_cmdline($pidfile, $pid)) {
2716                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2717                     return $pid;
2718                 }
2719             }
2720         }
2721     }
2722
2723     return undef;
2724 }
2725
2726 sub vzlist {
2727
2728     my $vzlist = config_list();
2729
2730     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2731
2732     while (defined(my $de = $fd->read)) {
2733         next if $de !~ m/^(\d+)\.pid$/;
2734         my $vmid = $1;
2735         next if !defined($vzlist->{$vmid});
2736         if (my $pid = check_running($vmid)) {
2737             $vzlist->{$vmid}->{pid} = $pid;
2738         }
2739     }
2740
2741     return $vzlist;
2742 }
2743
2744 sub disksize {
2745     my ($storecfg, $conf) = @_;
2746
2747     my $bootdisk = $conf->{bootdisk};
2748     return undef if !$bootdisk;
2749     return undef if !is_valid_drivename($bootdisk);
2750
2751     return undef if !$conf->{$bootdisk};
2752
2753     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2754     return undef if !defined($drive);
2755
2756     return undef if drive_is_cdrom($drive);
2757
2758     my $volid = $drive->{file};
2759     return undef if !$volid;
2760
2761     return $drive->{size};
2762 }
2763
2764 my $last_proc_pid_stat;
2765
2766 # get VM status information
2767 # This must be fast and should not block ($full == false)
2768 # We only query KVM using QMP if $full == true (this can be slow)
2769 sub vmstatus {
2770     my ($opt_vmid, $full) = @_;
2771
2772     my $res = {};
2773
2774     my $storecfg = PVE::Storage::config();
2775
2776     my $list = vzlist();
2777     my $defaults = load_defaults();
2778
2779     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2780
2781     my $cpucount = $cpuinfo->{cpus} || 1;
2782
2783     foreach my $vmid (keys %$list) {
2784         next if $opt_vmid && ($vmid ne $opt_vmid);
2785
2786         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2787         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2788
2789         my $d = {};
2790         $d->{pid} = $list->{$vmid}->{pid};
2791
2792         # fixme: better status?
2793         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2794
2795         my $size = disksize($storecfg, $conf);
2796         if (defined($size)) {
2797             $d->{disk} = 0; # no info available
2798             $d->{maxdisk} = $size;
2799         } else {
2800             $d->{disk} = 0;
2801             $d->{maxdisk} = 0;
2802         }
2803
2804         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2805             * ($conf->{cores} || $defaults->{cores});
2806         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2807         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2808
2809         $d->{name} = $conf->{name} || "VM $vmid";
2810         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2811             : $defaults->{memory}*(1024*1024);
2812
2813         if ($conf->{balloon}) {
2814             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2815             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2816                 : $defaults->{shares};
2817         }
2818
2819         $d->{uptime} = 0;
2820         $d->{cpu} = 0;
2821         $d->{mem} = 0;
2822
2823         $d->{netout} = 0;
2824         $d->{netin} = 0;
2825
2826         $d->{diskread} = 0;
2827         $d->{diskwrite} = 0;
2828
2829         $d->{template} = PVE::QemuConfig->is_template($conf);
2830
2831         $d->{serial} = 1 if conf_has_serial($conf);
2832
2833         $res->{$vmid} = $d;
2834     }
2835
2836     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2837     foreach my $dev (keys %$netdev) {
2838         next if $dev !~ m/^tap([1-9]\d*)i/;
2839         my $vmid = $1;
2840         my $d = $res->{$vmid};
2841         next if !$d;
2842
2843         $d->{netout} += $netdev->{$dev}->{receive};
2844         $d->{netin} += $netdev->{$dev}->{transmit};
2845
2846         if ($full) {
2847             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2848             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2849         }
2850
2851     }
2852
2853     my $ctime = gettimeofday;
2854
2855     foreach my $vmid (keys %$list) {
2856
2857         my $d = $res->{$vmid};
2858         my $pid = $d->{pid};
2859         next if !$pid;
2860
2861         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2862         next if !$pstat; # not running
2863
2864         my $used = $pstat->{utime} + $pstat->{stime};
2865
2866         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2867
2868         if ($pstat->{vsize}) {
2869             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2870         }
2871
2872         my $old = $last_proc_pid_stat->{$pid};
2873         if (!$old) {
2874             $last_proc_pid_stat->{$pid} = {
2875                 time => $ctime,
2876                 used => $used,
2877                 cpu => 0,
2878             };
2879             next;
2880         }
2881
2882         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2883
2884         if ($dtime > 1000) {
2885             my $dutime = $used -  $old->{used};
2886
2887             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2888             $last_proc_pid_stat->{$pid} = {
2889                 time => $ctime,
2890                 used => $used,
2891                 cpu => $d->{cpu},
2892             };
2893         } else {
2894             $d->{cpu} = $old->{cpu};
2895         }
2896     }
2897
2898     return $res if !$full;
2899
2900     my $qmpclient = PVE::QMPClient->new();
2901
2902     my $ballooncb = sub {
2903         my ($vmid, $resp) = @_;
2904
2905         my $info = $resp->{'return'};
2906         return if !$info->{max_mem};
2907
2908         my $d = $res->{$vmid};
2909
2910         # use memory assigned to VM
2911         $d->{maxmem} = $info->{max_mem};
2912         $d->{balloon} = $info->{actual};
2913
2914         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2915             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2916             $d->{freemem} = $info->{free_mem};
2917         }
2918
2919         $d->{ballooninfo} = $info;
2920     };
2921
2922     my $blockstatscb = sub {
2923         my ($vmid, $resp) = @_;
2924         my $data = $resp->{'return'} || [];
2925         my $totalrdbytes = 0;
2926         my $totalwrbytes = 0;
2927
2928         for my $blockstat (@$data) {
2929             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2930             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2931
2932             $blockstat->{device} =~ s/drive-//;
2933             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2934         }
2935         $res->{$vmid}->{diskread} = $totalrdbytes;
2936         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2937     };
2938
2939     my $statuscb = sub {
2940         my ($vmid, $resp) = @_;
2941
2942         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2943         # this fails if ballon driver is not loaded, so this must be
2944         # the last commnand (following command are aborted if this fails).
2945         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2946
2947         my $status = 'unknown';
2948         if (!defined($status = $resp->{'return'}->{status})) {
2949             warn "unable to get VM status\n";
2950             return;
2951         }
2952
2953         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2954     };
2955
2956     foreach my $vmid (keys %$list) {
2957         next if $opt_vmid && ($vmid ne $opt_vmid);
2958         next if !$res->{$vmid}->{pid}; # not running
2959         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2960     }
2961
2962     $qmpclient->queue_execute(undef, 2);
2963
2964     foreach my $vmid (keys %$list) {
2965         next if $opt_vmid && ($vmid ne $opt_vmid);
2966         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2967     }
2968
2969     return $res;
2970 }
2971
2972 sub foreach_drive {
2973     my ($conf, $func, @param) = @_;
2974
2975     foreach my $ds (valid_drive_names()) {
2976         next if !defined($conf->{$ds});
2977
2978         my $drive = parse_drive($ds, $conf->{$ds});
2979         next if !$drive;
2980
2981         &$func($ds, $drive, @param);
2982     }
2983 }
2984
2985 sub foreach_volid {
2986     my ($conf, $func, @param) = @_;
2987
2988     my $volhash = {};
2989
2990     my $test_volid = sub {
2991         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2992
2993         return if !$volid;
2994
2995         $volhash->{$volid}->{cdrom} //= 1;
2996         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2997
2998         $volhash->{$volid}->{replicate} //= 0;
2999         $volhash->{$volid}->{replicate} = 1 if $replicate;
3000
3001         $volhash->{$volid}->{shared} //= 0;
3002         $volhash->{$volid}->{shared} = 1 if $shared;
3003
3004         $volhash->{$volid}->{referenced_in_config} //= 0;
3005         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3006
3007         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3008             if defined($snapname);
3009     };
3010
3011     foreach_drive($conf, sub {
3012         my ($ds, $drive) = @_;
3013         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3014     });
3015
3016     foreach my $snapname (keys %{$conf->{snapshots}}) {
3017         my $snap = $conf->{snapshots}->{$snapname};
3018         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3019         foreach_drive($snap, sub {
3020             my ($ds, $drive) = @_;
3021             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3022         });
3023     }
3024
3025     foreach my $volid (keys %$volhash) {
3026         &$func($volid, $volhash->{$volid}, @param);
3027     }
3028 }
3029
3030 sub conf_has_serial {
3031     my ($conf) = @_;
3032
3033     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3034         if ($conf->{"serial$i"}) {
3035             return 1;
3036         }
3037     }
3038
3039     return 0;
3040 }
3041
3042 sub vga_conf_has_spice {
3043     my ($vga) = @_;
3044
3045     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3046
3047     return $1 || 1;
3048 }
3049
3050 sub config_to_command {
3051     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3052
3053     my $cmd = [];
3054     my $globalFlags = [];
3055     my $machineFlags = [];
3056     my $rtcFlags = [];
3057     my $cpuFlags = [];
3058     my $devices = [];
3059     my $pciaddr = '';
3060     my $bridges = {};
3061     my $kvmver = kvm_user_version();
3062     my $vernum = 0; # unknown
3063     my $ostype = $conf->{ostype};
3064     my $winversion = windows_version($ostype);
3065     my $kvm = $conf->{kvm} // 1;
3066
3067     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3068
3069     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3070         $vernum = $1*1000000+$2*1000;
3071     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3072         $vernum = $1*1000000+$2*1000+$3;
3073     }
3074
3075     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3076
3077     my $have_ovz = -f '/proc/vz/vestat';
3078
3079     my $q35 = machine_type_is_q35($conf);
3080     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3081     my $machine_type = $forcemachine || $conf->{machine};
3082     my $use_old_bios_files = undef;
3083     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3084
3085     my $cpuunits = defined($conf->{cpuunits}) ?
3086             $conf->{cpuunits} : $defaults->{cpuunits};
3087
3088     push @$cmd, '/usr/bin/kvm';
3089
3090     push @$cmd, '-id', $vmid;
3091
3092     my $use_virtio = 0;
3093
3094     my $qmpsocket = qmp_socket($vmid);
3095     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3096     push @$cmd, '-mon', "chardev=qmp,mode=control";
3097
3098
3099     push @$cmd, '-pidfile' , pidfile_name($vmid);
3100
3101     push @$cmd, '-daemonize';
3102
3103     if ($conf->{smbios1}) {
3104         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3105     }
3106
3107     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3108         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3109
3110         my $path;
3111         my $format;
3112         if (my $efidisk = $conf->{efidisk0}) {
3113             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3114             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3115             $format = $d->{format};
3116             if ($storeid) {
3117                 $path = PVE::Storage::path($storecfg, $d->{file});
3118                 if (!defined($format)) {
3119                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3120                     $format = qemu_img_format($scfg, $volname);
3121                 }
3122             } else {
3123                 $path = $d->{file};
3124                 die "efidisk format must be specified\n"
3125                     if !defined($format);
3126             }
3127         } else {
3128             warn "no efidisk configured! Using temporary efivars disk.\n";
3129             $path = "/tmp/$vmid-ovmf.fd";
3130             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3131             $format = 'raw';
3132         }
3133
3134         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3135         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3136     }
3137
3138
3139     # add usb controllers
3140     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3141     push @$devices, @usbcontrollers if @usbcontrollers;
3142     my $vga = $conf->{vga};
3143
3144     my $qxlnum = vga_conf_has_spice($vga);
3145     $vga = 'qxl' if $qxlnum;
3146
3147     if (!$vga) {
3148         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3149             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3150         } else {
3151             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3152         }
3153     }
3154
3155     # enable absolute mouse coordinates (needed by vnc)
3156     my $tablet;
3157     if (defined($conf->{tablet})) {
3158         $tablet = $conf->{tablet};
3159     } else {
3160         $tablet = $defaults->{tablet};
3161         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3162         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3163     }
3164
3165     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3166
3167     my $kvm_off = 0;
3168     my $gpu_passthrough;
3169
3170     # host pci devices
3171     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3172         my $d = parse_hostpci($conf->{"hostpci$i"});
3173         next if !$d;
3174
3175         my $pcie = $d->{pcie};
3176         if($pcie){
3177             die "q35 machine model is not enabled" if !$q35;
3178             $pciaddr = print_pcie_addr("hostpci$i");
3179         }else{
3180             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3181         }
3182
3183         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3184         my $romfile = $d->{romfile};
3185
3186         my $xvga = '';
3187         if ($d->{'x-vga'}) {
3188             $xvga = ',x-vga=on';
3189             $kvm_off = 1;
3190             $vga = 'none';
3191             $gpu_passthrough = 1;
3192
3193             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3194                 $xvga = "";
3195             }
3196         }
3197         my $pcidevices = $d->{pciid};
3198         my $multifunction = 1 if @$pcidevices > 1;
3199
3200         my $j=0;
3201         foreach my $pcidevice (@$pcidevices) {
3202
3203             my $id = "hostpci$i";
3204             $id .= ".$j" if $multifunction;
3205             my $addr = $pciaddr;
3206             $addr .= ".$j" if $multifunction;
3207             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3208
3209             if($j == 0){
3210                 $devicestr .= "$rombar$xvga";
3211                 $devicestr .= ",multifunction=on" if $multifunction;
3212                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3213             }
3214
3215             push @$devices, '-device', $devicestr;
3216             $j++;
3217         }
3218     }
3219
3220     # usb devices
3221     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3222     push @$devices, @usbdevices if @usbdevices;
3223     # serial devices
3224     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3225         if (my $path = $conf->{"serial$i"}) {
3226             if ($path eq 'socket') {
3227                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3228                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3229                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3230             } else {
3231                 die "no such serial device\n" if ! -c $path;
3232                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3233                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3234             }
3235         }
3236     }
3237
3238     # parallel devices
3239     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3240         if (my $path = $conf->{"parallel$i"}) {
3241             die "no such parallel device\n" if ! -c $path;
3242             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3243             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3244             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3245         }
3246     }
3247
3248     my $vmname = $conf->{name} || "vm$vmid";
3249
3250     push @$cmd, '-name', $vmname;
3251
3252     my $sockets = 1;
3253     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3254     $sockets = $conf->{sockets} if  $conf->{sockets};
3255
3256     my $cores = $conf->{cores} || 1;
3257
3258     my $maxcpus = $sockets * $cores;
3259
3260     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3261
3262     my $allowed_vcpus = $cpuinfo->{cpus};
3263
3264     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3265         if ($allowed_vcpus < $maxcpus);
3266
3267     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3268
3269         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3270         for (my $i = 2; $i <= $vcpus; $i++)  {
3271             my $cpustr = print_cpu_device($conf,$i);
3272             push @$cmd, '-device', $cpustr;
3273         }
3274
3275     } else {
3276
3277         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3278     }
3279     push @$cmd, '-nodefaults';
3280
3281     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3282
3283     my $bootindex_hash = {};
3284     my $i = 1;
3285     foreach my $o (split(//, $bootorder)) {
3286         $bootindex_hash->{$o} = $i*100;
3287         $i++;
3288     }
3289
3290     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3291
3292     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3293
3294     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3295
3296     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3297
3298     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3299         my $socket = vnc_socket($vmid);
3300         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3301     } else {
3302         push @$cmd, '-nographic';
3303     }
3304
3305     # time drift fix
3306     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3307
3308     my $useLocaltime = $conf->{localtime};
3309
3310     if ($winversion >= 5) { # windows
3311         $useLocaltime = 1 if !defined($conf->{localtime});
3312
3313         # use time drift fix when acpi is enabled
3314         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3315             $tdf = 1 if !defined($conf->{tdf});
3316         }
3317     }
3318
3319     if ($winversion >= 6) {
3320         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3321         push @$cmd, '-no-hpet';
3322     }
3323
3324     push @$rtcFlags, 'driftfix=slew' if $tdf;
3325
3326     if (!$kvm) {
3327         push @$machineFlags, 'accel=tcg';
3328     }
3329
3330     if ($machine_type) {
3331         push @$machineFlags, "type=${machine_type}";
3332     }
3333
3334     if ($conf->{startdate}) {
3335         push @$rtcFlags, "base=$conf->{startdate}";
3336     } elsif ($useLocaltime) {
3337         push @$rtcFlags, 'base=localtime';
3338     }
3339
3340     my $cpu = $kvm ? "kvm64" : "qemu64";
3341     if (my $cputype = $conf->{cpu}) {
3342         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3343             or die "Cannot parse cpu description: $cputype\n";
3344         $cpu = $cpuconf->{cputype};
3345         $kvm_off = 1 if $cpuconf->{hidden};
3346
3347         if (defined(my $flags = $cpuconf->{flags})) {
3348             push @$cpuFlags, split(";", $flags);
3349         }
3350     }
3351
3352     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3353
3354     push @$cpuFlags , '-x2apic'
3355         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3356
3357     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3358
3359     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3360
3361     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3362
3363         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3364         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3365     }
3366
3367     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3368
3369     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3370
3371     push @$cpuFlags, 'kvm=off' if $kvm_off;
3372
3373     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3374         die "internal error"; # should not happen
3375
3376     push @$cpuFlags, "vendor=${cpu_vendor}"
3377         if $cpu_vendor ne 'default';
3378
3379     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3380
3381     push @$cmd, '-cpu', $cpu;
3382
3383     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3384
3385     push @$cmd, '-S' if $conf->{freeze};
3386
3387     # set keyboard layout
3388     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3389     push @$cmd, '-k', $kb if $kb;
3390
3391     # enable sound
3392     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3393     #push @$cmd, '-soundhw', 'es1370';
3394     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3395
3396     if($conf->{agent}) {
3397         my $qgasocket = qmp_socket($vmid, 1);
3398         my $pciaddr = print_pci_addr("qga0", $bridges);
3399         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3400         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3401         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3402     }
3403
3404     my $spice_port;
3405
3406     if ($qxlnum) {
3407         if ($qxlnum > 1) {
3408             if ($winversion){
3409                 for(my $i = 1; $i < $qxlnum; $i++){
3410                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3411                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3412                 }
3413             } else {
3414                 # assume other OS works like Linux
3415                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3416                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3417             }
3418         }
3419
3420         my $pciaddr = print_pci_addr("spice", $bridges);
3421
3422         my $nodename = PVE::INotify::nodename();
3423         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3424         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3425         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3426         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3427         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3428
3429         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3430
3431         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3432         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3433         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3434     }
3435
3436     # enable balloon by default, unless explicitly disabled
3437     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3438         $pciaddr = print_pci_addr("balloon0", $bridges);
3439         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3440     }
3441
3442     if ($conf->{watchdog}) {
3443         my $wdopts = parse_watchdog($conf->{watchdog});
3444         $pciaddr = print_pci_addr("watchdog", $bridges);
3445         my $watchdog = $wdopts->{model} || 'i6300esb';
3446         push @$devices, '-device', "$watchdog$pciaddr";
3447         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3448     }
3449
3450     my $vollist = [];
3451     my $scsicontroller = {};
3452     my $ahcicontroller = {};
3453     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3454
3455     # Add iscsi initiator name if available
3456     if (my $initiator = get_initiator_name()) {
3457         push @$devices, '-iscsi', "initiator-name=$initiator";
3458     }
3459
3460     foreach_drive($conf, sub {
3461         my ($ds, $drive) = @_;
3462
3463         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3464             push @$vollist, $drive->{file};
3465         }
3466
3467         # ignore efidisk here, already added in bios/fw handling code above
3468         return if $drive->{interface} eq 'efidisk';
3469
3470         $use_virtio = 1 if $ds =~ m/^virtio/;
3471
3472         if (drive_is_cdrom ($drive)) {
3473             if ($bootindex_hash->{d}) {
3474                 $drive->{bootindex} = $bootindex_hash->{d};
3475                 $bootindex_hash->{d} += 1;
3476             }
3477         } else {
3478             if ($bootindex_hash->{c}) {
3479                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3480                 $bootindex_hash->{c} += 1;
3481             }
3482         }
3483
3484         if($drive->{interface} eq 'virtio'){
3485            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3486         }
3487
3488         if ($drive->{interface} eq 'scsi') {
3489
3490             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3491
3492             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3493             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3494
3495             my $iothread = '';
3496             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3497                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3498                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3499             } elsif ($drive->{iothread}) {
3500                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3501             }
3502
3503             my $queues = '';
3504             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3505                 $queues = ",num_queues=$drive->{queues}";
3506             }
3507
3508             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3509             $scsicontroller->{$controller}=1;
3510         }
3511
3512         if ($drive->{interface} eq 'sata') {
3513            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3514            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3515            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3516            $ahcicontroller->{$controller}=1;
3517         }
3518
3519         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3520         push @$devices, '-drive',$drive_cmd;
3521         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3522     });
3523
3524     for (my $i = 0; $i < $MAX_NETS; $i++) {
3525          next if !$conf->{"net$i"};
3526          my $d = parse_net($conf->{"net$i"});
3527          next if !$d;
3528
3529          $use_virtio = 1 if $d->{model} eq 'virtio';
3530
3531          if ($bootindex_hash->{n}) {
3532             $d->{bootindex} = $bootindex_hash->{n};
3533             $bootindex_hash->{n} += 1;
3534          }
3535
3536          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3537          push @$devices, '-netdev', $netdevfull;
3538
3539          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3540          push @$devices, '-device', $netdevicefull;
3541     }
3542
3543     if (!$q35) {
3544         # add pci bridges
3545         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3546            $bridges->{1} = 1;
3547            $bridges->{2} = 1;
3548         }
3549
3550         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3551
3552         while (my ($k, $v) = each %$bridges) {
3553             $pciaddr = print_pci_addr("pci.$k");
3554             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3555         }
3556     }
3557
3558     # add custom args
3559     if ($conf->{args}) {
3560         my $aa = PVE::Tools::split_args($conf->{args});
3561         push @$cmd, @$aa;
3562     }
3563
3564     push @$cmd, @$devices;
3565     push @$cmd, '-rtc', join(',', @$rtcFlags)
3566         if scalar(@$rtcFlags);
3567     push @$cmd, '-machine', join(',', @$machineFlags)
3568         if scalar(@$machineFlags);
3569     push @$cmd, '-global', join(',', @$globalFlags)
3570         if scalar(@$globalFlags);
3571
3572     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3573 }
3574
3575 sub vnc_socket {
3576     my ($vmid) = @_;
3577     return "${var_run_tmpdir}/$vmid.vnc";
3578 }
3579
3580 sub spice_port {
3581     my ($vmid) = @_;
3582
3583     my $res = vm_mon_cmd($vmid, 'query-spice');
3584
3585     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3586 }
3587
3588 sub qmp_socket {
3589     my ($vmid, $qga) = @_;
3590     my $sockettype = $qga ? 'qga' : 'qmp';
3591     return "${var_run_tmpdir}/$vmid.$sockettype";
3592 }
3593
3594 sub pidfile_name {
3595     my ($vmid) = @_;
3596     return "${var_run_tmpdir}/$vmid.pid";
3597 }
3598
3599 sub vm_devices_list {
3600     my ($vmid) = @_;
3601
3602     my $res = vm_mon_cmd($vmid, 'query-pci');
3603     my $devices = {};
3604     foreach my $pcibus (@$res) {
3605         foreach my $device (@{$pcibus->{devices}}) {
3606             next if !$device->{'qdev_id'};
3607             if ($device->{'pci_bridge'}) {
3608                 $devices->{$device->{'qdev_id'}} = 1;
3609                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3610                     next if !$bridge_device->{'qdev_id'};
3611                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3612                     $devices->{$device->{'qdev_id'}}++;
3613                 }
3614             } else {
3615                 $devices->{$device->{'qdev_id'}} = 1;
3616             }
3617         }
3618     }
3619
3620     my $resblock = vm_mon_cmd($vmid, 'query-block');
3621     foreach my $block (@$resblock) {
3622         if($block->{device} =~ m/^drive-(\S+)/){
3623                 $devices->{$1} = 1;
3624         }
3625     }
3626
3627     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3628     foreach my $mice (@$resmice) {
3629         if ($mice->{name} eq 'QEMU HID Tablet') {
3630             $devices->{tablet} = 1;
3631             last;
3632         }
3633     }
3634
3635     # for usb devices there is no query-usb
3636     # but we can iterate over the entries in
3637     # qom-list path=/machine/peripheral
3638     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3639     foreach my $per (@$resperipheral) {
3640         if ($per->{name} =~ m/^usb\d+$/) {
3641             $devices->{$per->{name}} = 1;
3642         }
3643     }
3644
3645     return $devices;
3646 }
3647
3648 sub vm_deviceplug {
3649     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3650
3651     my $q35 = machine_type_is_q35($conf);
3652
3653     my $devices_list = vm_devices_list($vmid);
3654     return 1 if defined($devices_list->{$deviceid});
3655
3656     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3657
3658     if ($deviceid eq 'tablet') {
3659
3660         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3661
3662     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3663
3664         die "usb hotplug currently not reliable\n";
3665         # since we can't reliably hot unplug all added usb devices
3666         # and usb passthrough disables live migration
3667         # we disable usb hotplugging for now
3668         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3669
3670     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3671
3672         qemu_iothread_add($vmid, $deviceid, $device);
3673
3674         qemu_driveadd($storecfg, $vmid, $device);
3675         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3676
3677         qemu_deviceadd($vmid, $devicefull);
3678         eval { qemu_deviceaddverify($vmid, $deviceid); };
3679         if (my $err = $@) {
3680             eval { qemu_drivedel($vmid, $deviceid); };
3681             warn $@ if $@;
3682             die $err;
3683         }
3684
3685     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3686
3687
3688         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3689         my $pciaddr = print_pci_addr($deviceid);
3690         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3691
3692         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3693
3694         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3695             qemu_iothread_add($vmid, $deviceid, $device);
3696             $devicefull .= ",iothread=iothread-$deviceid";
3697         }
3698
3699         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3700             $devicefull .= ",num_queues=$device->{queues}";
3701         }
3702
3703         qemu_deviceadd($vmid, $devicefull);
3704         qemu_deviceaddverify($vmid, $deviceid);
3705
3706     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3707
3708         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3709         qemu_driveadd($storecfg, $vmid, $device);
3710
3711         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3712         eval { qemu_deviceadd($vmid, $devicefull); };
3713         if (my $err = $@) {
3714             eval { qemu_drivedel($vmid, $deviceid); };
3715             warn $@ if $@;
3716             die $err;
3717         }
3718
3719     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3720
3721         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3722
3723         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3724         my $use_old_bios_files = undef;
3725         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3726
3727         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3728         qemu_deviceadd($vmid, $netdevicefull);
3729         eval { qemu_deviceaddverify($vmid, $deviceid); };
3730         if (my $err = $@) {
3731             eval { qemu_netdevdel($vmid, $deviceid); };
3732             warn $@ if $@;
3733             die $err;
3734         }
3735
3736     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3737
3738         my $bridgeid = $2;
3739         my $pciaddr = print_pci_addr($deviceid);
3740         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3741
3742         qemu_deviceadd($vmid, $devicefull);
3743         qemu_deviceaddverify($vmid, $deviceid);
3744
3745     } else {
3746         die "can't hotplug device '$deviceid'\n";
3747     }
3748
3749     return 1;
3750 }
3751
3752 # fixme: this should raise exceptions on error!
3753 sub vm_deviceunplug {
3754     my ($vmid, $conf, $deviceid) = @_;
3755
3756     my $devices_list = vm_devices_list($vmid);
3757     return 1 if !defined($devices_list->{$deviceid});
3758
3759     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3760
3761     if ($deviceid eq 'tablet') {
3762
3763         qemu_devicedel($vmid, $deviceid);
3764
3765     } elsif ($deviceid =~ m/^usb\d+$/) {
3766
3767         die "usb hotplug currently not reliable\n";
3768         # when unplugging usb devices this way,
3769         # there may be remaining usb controllers/hubs
3770         # so we disable it for now
3771         qemu_devicedel($vmid, $deviceid);
3772         qemu_devicedelverify($vmid, $deviceid);
3773
3774     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3775
3776         qemu_devicedel($vmid, $deviceid);
3777         qemu_devicedelverify($vmid, $deviceid);
3778         qemu_drivedel($vmid, $deviceid);
3779         qemu_iothread_del($conf, $vmid, $deviceid);
3780
3781     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3782
3783         qemu_devicedel($vmid, $deviceid);
3784         qemu_devicedelverify($vmid, $deviceid);
3785         qemu_iothread_del($conf, $vmid, $deviceid);
3786
3787     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3788
3789         qemu_devicedel($vmid, $deviceid);
3790         qemu_drivedel($vmid, $deviceid);
3791         qemu_deletescsihw($conf, $vmid, $deviceid);
3792
3793     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3794
3795         qemu_devicedel($vmid, $deviceid);
3796         qemu_devicedelverify($vmid, $deviceid);
3797         qemu_netdevdel($vmid, $deviceid);
3798
3799     } else {
3800         die "can't unplug device '$deviceid'\n";
3801     }
3802
3803     return 1;
3804 }
3805
3806 sub qemu_deviceadd {
3807     my ($vmid, $devicefull) = @_;
3808
3809     $devicefull = "driver=".$devicefull;
3810     my %options =  split(/[=,]/, $devicefull);
3811
3812     vm_mon_cmd($vmid, "device_add" , %options);
3813 }
3814
3815 sub qemu_devicedel {
3816     my ($vmid, $deviceid) = @_;
3817
3818     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3819 }
3820
3821 sub qemu_iothread_add {
3822     my($vmid, $deviceid, $device) = @_;
3823
3824     if ($device->{iothread}) {
3825         my $iothreads = vm_iothreads_list($vmid);
3826         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3827     }
3828 }
3829
3830 sub qemu_iothread_del {
3831     my($conf, $vmid, $deviceid) = @_;
3832
3833     my $device = parse_drive($deviceid, $conf->{$deviceid});
3834     if ($device->{iothread}) {
3835         my $iothreads = vm_iothreads_list($vmid);
3836         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3837     }
3838 }
3839
3840 sub qemu_objectadd {
3841     my($vmid, $objectid, $qomtype) = @_;
3842
3843     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3844
3845     return 1;
3846 }
3847
3848 sub qemu_objectdel {
3849     my($vmid, $objectid) = @_;
3850
3851     vm_mon_cmd($vmid, "object-del", id => $objectid);
3852
3853     return 1;
3854 }
3855
3856 sub qemu_driveadd {
3857     my ($storecfg, $vmid, $device) = @_;
3858
3859     my $drive = print_drive_full($storecfg, $vmid, $device);
3860     $drive =~ s/\\/\\\\/g;
3861     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3862
3863     # If the command succeeds qemu prints: "OK"
3864     return 1 if $ret =~ m/OK/s;
3865
3866     die "adding drive failed: $ret\n";
3867 }
3868
3869 sub qemu_drivedel {
3870     my($vmid, $deviceid) = @_;
3871
3872     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3873     $ret =~ s/^\s+//;
3874
3875     return 1 if $ret eq "";
3876
3877     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3878     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3879
3880     die "deleting drive $deviceid failed : $ret\n";
3881 }
3882
3883 sub qemu_deviceaddverify {
3884     my ($vmid, $deviceid) = @_;
3885
3886     for (my $i = 0; $i <= 5; $i++) {
3887          my $devices_list = vm_devices_list($vmid);
3888          return 1 if defined($devices_list->{$deviceid});
3889          sleep 1;
3890     }
3891
3892     die "error on hotplug device '$deviceid'\n";
3893 }
3894
3895
3896 sub qemu_devicedelverify {
3897     my ($vmid, $deviceid) = @_;
3898
3899     # need to verify that the device is correctly removed as device_del
3900     # is async and empty return is not reliable
3901
3902     for (my $i = 0; $i <= 5; $i++) {
3903          my $devices_list = vm_devices_list($vmid);
3904          return 1 if !defined($devices_list->{$deviceid});
3905          sleep 1;
3906     }
3907
3908     die "error on hot-unplugging device '$deviceid'\n";
3909 }
3910
3911 sub qemu_findorcreatescsihw {
3912     my ($storecfg, $conf, $vmid, $device) = @_;
3913
3914     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3915
3916     my $scsihwid="$controller_prefix$controller";
3917     my $devices_list = vm_devices_list($vmid);
3918
3919     if(!defined($devices_list->{$scsihwid})) {
3920         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3921     }
3922
3923     return 1;
3924 }
3925
3926 sub qemu_deletescsihw {
3927     my ($conf, $vmid, $opt) = @_;
3928
3929     my $device = parse_drive($opt, $conf->{$opt});
3930
3931     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3932         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3933         return 1;
3934     }
3935
3936     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3937
3938     my $devices_list = vm_devices_list($vmid);
3939     foreach my $opt (keys %{$devices_list}) {
3940         if (PVE::QemuServer::is_valid_drivename($opt)) {
3941             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3942             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3943                 return 1;
3944             }
3945         }
3946     }
3947
3948     my $scsihwid="scsihw$controller";
3949
3950     vm_deviceunplug($vmid, $conf, $scsihwid);
3951
3952     return 1;
3953 }
3954
3955 sub qemu_add_pci_bridge {
3956     my ($storecfg, $conf, $vmid, $device) = @_;
3957
3958     my $bridges = {};
3959
3960     my $bridgeid;
3961
3962     print_pci_addr($device, $bridges);
3963
3964     while (my ($k, $v) = each %$bridges) {
3965         $bridgeid = $k;
3966     }
3967     return 1 if !defined($bridgeid) || $bridgeid < 1;
3968
3969     my $bridge = "pci.$bridgeid";
3970     my $devices_list = vm_devices_list($vmid);
3971
3972     if (!defined($devices_list->{$bridge})) {
3973         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3974     }
3975
3976     return 1;
3977 }
3978
3979 sub qemu_set_link_status {
3980     my ($vmid, $device, $up) = @_;
3981
3982     vm_mon_cmd($vmid, "set_link", name => $device,
3983                up => $up ? JSON::true : JSON::false);
3984 }
3985
3986 sub qemu_netdevadd {
3987     my ($vmid, $conf, $device, $deviceid) = @_;
3988
3989     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3990     my %options =  split(/[=,]/, $netdev);
3991
3992     vm_mon_cmd($vmid, "netdev_add",  %options);
3993     return 1;
3994 }
3995
3996 sub qemu_netdevdel {
3997     my ($vmid, $deviceid) = @_;
3998
3999     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4000 }
4001
4002 sub qemu_usb_hotplug {
4003     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4004
4005     return if !$device;
4006
4007     # remove the old one first
4008     vm_deviceunplug($vmid, $conf, $deviceid);
4009
4010     # check if xhci controller is necessary and available
4011     if ($device->{usb3}) {
4012
4013         my $devicelist = vm_devices_list($vmid);
4014
4015         if (!$devicelist->{xhci}) {
4016             my $pciaddr = print_pci_addr("xhci");
4017             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4018         }
4019     }
4020     my $d = parse_usb_device($device->{host});
4021     $d->{usb3} = $device->{usb3};
4022
4023     # add the new one
4024     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4025 }
4026
4027 sub qemu_cpu_hotplug {
4028     my ($vmid, $conf, $vcpus) = @_;
4029
4030     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4031
4032     my $sockets = 1;
4033     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4034     $sockets = $conf->{sockets} if  $conf->{sockets};
4035     my $cores = $conf->{cores} || 1;
4036     my $maxcpus = $sockets * $cores;
4037
4038     $vcpus = $maxcpus if !$vcpus;
4039
4040     die "you can't add more vcpus than maxcpus\n"
4041         if $vcpus > $maxcpus;
4042
4043     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4044
4045     if ($vcpus < $currentvcpus) {
4046
4047         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4048
4049             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4050                 qemu_devicedel($vmid, "cpu$i");
4051                 my $retry = 0;
4052                 my $currentrunningvcpus = undef;
4053                 while (1) {
4054                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4055                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4056                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4057                     $retry++;
4058                     sleep 1;
4059                 }
4060                 #update conf after each succesfull cpu unplug
4061                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4062                 PVE::QemuConfig->write_config($vmid, $conf);
4063             }
4064         } else {
4065             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4066         }
4067
4068         return;
4069     }
4070
4071     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4072     die "vcpus in running vm does not match its configuration\n"
4073         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4074
4075     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4076
4077         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4078             my $cpustr = print_cpu_device($conf, $i);
4079             qemu_deviceadd($vmid, $cpustr);
4080
4081             my $retry = 0;
4082             my $currentrunningvcpus = undef;
4083             while (1) {
4084                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4085                 last if scalar(@{$currentrunningvcpus}) == $i;
4086                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4087                 sleep 1;
4088                 $retry++;
4089             }
4090             #update conf after each succesfull cpu hotplug
4091             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4092             PVE::QemuConfig->write_config($vmid, $conf);
4093         }
4094     } else {
4095
4096         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4097             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4098         }
4099     }
4100 }
4101
4102 sub qemu_block_set_io_throttle {
4103     my ($vmid, $deviceid,
4104         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4105         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4106         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4107         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4108
4109     return if !check_running($vmid) ;
4110
4111     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4112         bps => int($bps),
4113         bps_rd => int($bps_rd),
4114         bps_wr => int($bps_wr),
4115         iops => int($iops),
4116         iops_rd => int($iops_rd),
4117         iops_wr => int($iops_wr),
4118         bps_max => int($bps_max),
4119         bps_rd_max => int($bps_rd_max),
4120         bps_wr_max => int($bps_wr_max),
4121         iops_max => int($iops_max),
4122         iops_rd_max => int($iops_rd_max),
4123         iops_wr_max => int($iops_wr_max),
4124         bps_max_length => int($bps_max_length),
4125         bps_rd_max_length => int($bps_rd_max_length),
4126         bps_wr_max_length => int($bps_wr_max_length),
4127         iops_max_length => int($iops_max_length),
4128         iops_rd_max_length => int($iops_rd_max_length),
4129         iops_wr_max_length => int($iops_wr_max_length),
4130     );
4131
4132 }
4133
4134 # old code, only used to shutdown old VM after update
4135 sub __read_avail {
4136     my ($fh, $timeout) = @_;
4137
4138     my $sel = new IO::Select;
4139     $sel->add($fh);
4140
4141     my $res = '';
4142     my $buf;
4143
4144     my @ready;
4145     while (scalar (@ready = $sel->can_read($timeout))) {
4146         my $count;
4147         if ($count = $fh->sysread($buf, 8192)) {
4148             if ($buf =~ /^(.*)\(qemu\) $/s) {
4149                 $res .= $1;
4150                 last;
4151             } else {
4152                 $res .= $buf;
4153             }
4154         } else {
4155             if (!defined($count)) {
4156                 die "$!\n";
4157             }
4158             last;
4159         }
4160     }
4161
4162     die "monitor read timeout\n" if !scalar(@ready);
4163
4164     return $res;
4165 }
4166
4167 # old code, only used to shutdown old VM after update
4168 sub vm_monitor_command {
4169     my ($vmid, $cmdstr, $nocheck) = @_;
4170
4171     my $res;
4172
4173     eval {
4174         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4175
4176         my $sname = "${var_run_tmpdir}/$vmid.mon";
4177
4178         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4179             die "unable to connect to VM $vmid socket - $!\n";
4180
4181         my $timeout = 3;
4182
4183         # hack: migrate sometime blocks the monitor (when migrate_downtime
4184         # is set)
4185         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4186             $timeout = 60*60; # 1 hour
4187         }
4188
4189         # read banner;
4190         my $data = __read_avail($sock, $timeout);
4191
4192         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4193             die "got unexpected qemu monitor banner\n";
4194         }
4195
4196         my $sel = new IO::Select;
4197         $sel->add($sock);
4198
4199         if (!scalar(my @ready = $sel->can_write($timeout))) {
4200             die "monitor write error - timeout";
4201         }
4202
4203         my $fullcmd = "$cmdstr\r";
4204
4205         # syslog('info', "VM $vmid monitor command: $cmdstr");
4206
4207         my $b;
4208         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4209             die "monitor write error - $!";
4210         }
4211
4212         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4213
4214         $timeout = 20;
4215
4216         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4217             $timeout = 60*60; # 1 hour
4218         } elsif ($cmdstr =~ m/^(eject|change)/) {
4219             $timeout = 60; # note: cdrom mount command is slow
4220         }
4221         if ($res = __read_avail($sock, $timeout)) {
4222
4223             my @lines = split("\r?\n", $res);
4224
4225             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4226
4227             $res = join("\n", @lines);
4228             $res .= "\n";
4229         }
4230     };
4231
4232     my $err = $@;
4233
4234     if ($err) {
4235         syslog("err", "VM $vmid monitor command failed - $err");
4236         die $err;
4237     }
4238
4239     return $res;
4240 }
4241
4242 sub qemu_block_resize {
4243     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4244
4245     my $running = check_running($vmid);
4246
4247     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4248
4249     return if !$running;
4250
4251     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4252
4253 }
4254
4255 sub qemu_volume_snapshot {
4256     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4257
4258     my $running = check_running($vmid);
4259
4260     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4261         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4262     } else {
4263         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4264     }
4265 }
4266
4267 sub qemu_volume_snapshot_delete {
4268     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4269
4270     my $running = check_running($vmid);
4271
4272     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4273         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4274     } else {
4275         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4276     }
4277 }
4278
4279 sub set_migration_caps {
4280     my ($vmid) = @_;
4281
4282     my $cap_ref = [];
4283
4284     my $enabled_cap = {
4285         "auto-converge" => 1,
4286         "xbzrle" => 1,
4287         "x-rdma-pin-all" => 0,
4288         "zero-blocks" => 0,
4289         "compress" => 0
4290     };
4291
4292     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4293
4294     for my $supported_capability (@$supported_capabilities) {
4295         push @$cap_ref, {
4296             capability => $supported_capability->{capability},
4297             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4298         };
4299     }
4300
4301     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4302 }
4303
4304 my $fast_plug_option = {
4305     'lock' => 1,
4306     'name' => 1,
4307     'onboot' => 1,
4308     'shares' => 1,
4309     'startup' => 1,
4310     'description' => 1,
4311     'protection' => 1,
4312     'vmstatestorage' => 1,
4313 };
4314
4315 # hotplug changes in [PENDING]
4316 # $selection hash can be used to only apply specified options, for
4317 # example: { cores => 1 } (only apply changed 'cores')
4318 # $errors ref is used to return error messages
4319 sub vmconfig_hotplug_pending {
4320     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4321
4322     my $defaults = load_defaults();
4323
4324     # commit values which do not have any impact on running VM first
4325     # Note: those option cannot raise errors, we we do not care about
4326     # $selection and always apply them.
4327
4328     my $add_error = sub {
4329         my ($opt, $msg) = @_;
4330         $errors->{$opt} = "hotplug problem - $msg";
4331     };
4332
4333     my $changes = 0;
4334     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4335         if ($fast_plug_option->{$opt}) {
4336             $conf->{$opt} = $conf->{pending}->{$opt};
4337             delete $conf->{pending}->{$opt};
4338             $changes = 1;
4339         }
4340     }
4341
4342     if ($changes) {
4343         PVE::QemuConfig->write_config($vmid, $conf);
4344         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4345     }
4346
4347     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4348
4349     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4350     while (my ($opt, $force) = each %$pending_delete_hash) {
4351         next if $selection && !$selection->{$opt};
4352         eval {
4353             if ($opt eq 'hotplug') {
4354                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4355             } elsif ($opt eq 'tablet') {
4356                 die "skip\n" if !$hotplug_features->{usb};
4357                 if ($defaults->{tablet}) {
4358                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4359                 } else {
4360                     vm_deviceunplug($vmid, $conf, $opt);
4361                 }
4362             } elsif ($opt =~ m/^usb\d+/) {
4363                 die "skip\n";
4364                 # since we cannot reliably hot unplug usb devices
4365                 # we are disabling it
4366                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4367                 vm_deviceunplug($vmid, $conf, $opt);
4368             } elsif ($opt eq 'vcpus') {
4369                 die "skip\n" if !$hotplug_features->{cpu};
4370                 qemu_cpu_hotplug($vmid, $conf, undef);
4371             } elsif ($opt eq 'balloon') {
4372                 # enable balloon device is not hotpluggable
4373                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4374             } elsif ($fast_plug_option->{$opt}) {
4375                 # do nothing
4376             } elsif ($opt =~ m/^net(\d+)$/) {
4377                 die "skip\n" if !$hotplug_features->{network};
4378                 vm_deviceunplug($vmid, $conf, $opt);
4379             } elsif (is_valid_drivename($opt)) {
4380                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4381                 vm_deviceunplug($vmid, $conf, $opt);
4382                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4383             } elsif ($opt =~ m/^memory$/) {
4384                 die "skip\n" if !$hotplug_features->{memory};
4385                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4386             } elsif ($opt eq 'cpuunits') {
4387                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4388             } elsif ($opt eq 'cpulimit') {
4389                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4390             } else {
4391                 die "skip\n";
4392             }
4393         };
4394         if (my $err = $@) {
4395             &$add_error($opt, $err) if $err ne "skip\n";
4396         } else {
4397             # save new config if hotplug was successful
4398             delete $conf->{$opt};
4399             vmconfig_undelete_pending_option($conf, $opt);
4400             PVE::QemuConfig->write_config($vmid, $conf);
4401             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4402         }
4403     }
4404
4405     my $apply_pending_cloudinit;
4406     $apply_pending_cloudinit = sub {
4407         my ($key, $value) = @_;
4408         $apply_pending_cloudinit = sub {}; # once is enough
4409
4410         my @cloudinit_opts = keys %$confdesc_cloudinit;
4411         foreach my $opt (keys %{$conf->{pending}}) {
4412             next if !grep { $_ eq $opt } @cloudinit_opts;
4413             $conf->{$opt} = delete $conf->{pending}->{$opt};
4414         }
4415
4416         my $new_conf = { %$conf };
4417         $new_conf->{$key} = $value;
4418         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4419     };
4420
4421     foreach my $opt (keys %{$conf->{pending}}) {
4422         next if $selection && !$selection->{$opt};
4423         my $value = $conf->{pending}->{$opt};
4424         eval {
4425             if ($opt eq 'hotplug') {
4426                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4427             } elsif ($opt eq 'tablet') {
4428                 die "skip\n" if !$hotplug_features->{usb};
4429                 if ($value == 1) {
4430                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4431                 } elsif ($value == 0) {
4432                     vm_deviceunplug($vmid, $conf, $opt);
4433                 }
4434             } elsif ($opt =~ m/^usb\d+$/) {
4435                 die "skip\n";
4436                 # since we cannot reliably hot unplug usb devices
4437                 # we are disabling it
4438                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4439                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4440                 die "skip\n" if !$d;
4441                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4442             } elsif ($opt eq 'vcpus') {
4443                 die "skip\n" if !$hotplug_features->{cpu};
4444                 qemu_cpu_hotplug($vmid, $conf, $value);
4445             } elsif ($opt eq 'balloon') {
4446                 # enable/disable balloning device is not hotpluggable
4447                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4448                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4449                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4450
4451                 # allow manual ballooning if shares is set to zero
4452                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4453                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4454                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4455                 }
4456             } elsif ($opt =~ m/^net(\d+)$/) {
4457                 # some changes can be done without hotplug
4458                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4459                                     $vmid, $opt, $value);
4460             } elsif (is_valid_drivename($opt)) {
4461                 # some changes can be done without hotplug
4462                 my $drive = parse_drive($opt, $value);
4463                 if (drive_is_cloudinit($drive)) {
4464                     &$apply_pending_cloudinit($opt, $value);
4465                 }
4466                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4467                                      $vmid, $opt, $value, 1);
4468             } elsif ($opt =~ m/^memory$/) { #dimms
4469                 die "skip\n" if !$hotplug_features->{memory};
4470                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4471             } elsif ($opt eq 'cpuunits') {
4472                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4473             } elsif ($opt eq 'cpulimit') {
4474                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4475                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4476             } else {
4477                 die "skip\n";  # skip non-hot-pluggable options
4478             }
4479         };
4480         if (my $err = $@) {
4481             &$add_error($opt, $err) if $err ne "skip\n";
4482         } else {
4483             # save new config if hotplug was successful
4484             $conf->{$opt} = $value;
4485             delete $conf->{pending}->{$opt};
4486             PVE::QemuConfig->write_config($vmid, $conf);
4487             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4488         }
4489     }
4490 }
4491
4492 sub try_deallocate_drive {
4493     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4494
4495     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4496         my $volid = $drive->{file};
4497         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4498             my $sid = PVE::Storage::parse_volume_id($volid);
4499             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4500
4501             # check if the disk is really unused
4502             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4503                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4504             PVE::Storage::vdisk_free($storecfg, $volid);
4505             return 1;
4506         } else {
4507             # If vm is not owner of this disk remove from config
4508             return 1;
4509         }
4510     }
4511
4512     return undef;
4513 }
4514
4515 sub vmconfig_delete_or_detach_drive {
4516     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4517
4518     my $drive = parse_drive($opt, $conf->{$opt});
4519
4520     my $rpcenv = PVE::RPCEnvironment::get();
4521     my $authuser = $rpcenv->get_user();
4522
4523     if ($force) {
4524         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4525         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4526     } else {
4527         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4528     }
4529 }
4530
4531 sub vmconfig_apply_pending {
4532     my ($vmid, $conf, $storecfg) = @_;
4533
4534     # cold plug
4535
4536     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4537     while (my ($opt, $force) = each %$pending_delete_hash) {
4538         die "internal error" if $opt =~ m/^unused/;
4539         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4540         if (!defined($conf->{$opt})) {
4541             vmconfig_undelete_pending_option($conf, $opt);
4542             PVE::QemuConfig->write_config($vmid, $conf);
4543         } elsif (is_valid_drivename($opt)) {
4544             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4545             vmconfig_undelete_pending_option($conf, $opt);
4546             delete $conf->{$opt};
4547             PVE::QemuConfig->write_config($vmid, $conf);
4548         } else {
4549             vmconfig_undelete_pending_option($conf, $opt);
4550             delete $conf->{$opt};
4551             PVE::QemuConfig->write_config($vmid, $conf);
4552         }
4553     }
4554
4555     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4556
4557     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4558         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4559
4560         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4561             # skip if nothing changed
4562         } elsif (is_valid_drivename($opt)) {
4563             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4564                 if defined($conf->{$opt});
4565             $conf->{$opt} = $conf->{pending}->{$opt};
4566         } else {
4567             $conf->{$opt} = $conf->{pending}->{$opt};
4568         }
4569
4570         delete $conf->{pending}->{$opt};
4571         PVE::QemuConfig->write_config($vmid, $conf);
4572     }
4573 }
4574
4575 my $safe_num_ne = sub {
4576     my ($a, $b) = @_;
4577
4578     return 0 if !defined($a) && !defined($b);
4579     return 1 if !defined($a);
4580     return 1 if !defined($b);
4581
4582     return $a != $b;
4583 };
4584
4585 my $safe_string_ne = sub {
4586     my ($a, $b) = @_;
4587
4588     return 0 if !defined($a) && !defined($b);
4589     return 1 if !defined($a);
4590     return 1 if !defined($b);
4591
4592     return $a ne $b;
4593 };
4594
4595 sub vmconfig_update_net {
4596     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4597
4598     my $newnet = parse_net($value);
4599
4600     if ($conf->{$opt}) {
4601         my $oldnet = parse_net($conf->{$opt});
4602
4603         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4604             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4605             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4606             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4607
4608             # for non online change, we try to hot-unplug
4609             die "skip\n" if !$hotplug;
4610             vm_deviceunplug($vmid, $conf, $opt);
4611         } else {
4612
4613             die "internal error" if $opt !~ m/net(\d+)/;
4614             my $iface = "tap${vmid}i$1";
4615
4616             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4617                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4618                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4619                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4620                 PVE::Network::tap_unplug($iface);
4621                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4622             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4623                 # Rate can be applied on its own but any change above needs to
4624                 # include the rate in tap_plug since OVS resets everything.
4625                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4626             }
4627
4628             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4629                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4630             }
4631
4632             return 1;
4633         }
4634     }
4635
4636     if ($hotplug) {
4637         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4638     } else {
4639         die "skip\n";
4640     }
4641 }
4642
4643 sub vmconfig_update_disk {
4644     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4645
4646     # fixme: do we need force?
4647
4648     my $drive = parse_drive($opt, $value);
4649
4650     if ($conf->{$opt}) {
4651
4652         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4653
4654             my $media = $drive->{media} || 'disk';
4655             my $oldmedia = $old_drive->{media} || 'disk';
4656             die "unable to change media type\n" if $media ne $oldmedia;
4657
4658             if (!drive_is_cdrom($old_drive)) {
4659
4660                 if ($drive->{file} ne $old_drive->{file}) {
4661
4662                     die "skip\n" if !$hotplug;
4663
4664                     # unplug and register as unused
4665                     vm_deviceunplug($vmid, $conf, $opt);
4666                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4667
4668                 } else {
4669                     # update existing disk
4670
4671                     # skip non hotpluggable value
4672                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4673                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4674                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4675                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4676                         die "skip\n";
4677                     }
4678
4679                     # apply throttle
4680                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4681                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4682                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4683                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4684                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4685                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4686                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4687                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4688                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4689                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4690                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4691                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4692                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4693                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4694                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4695                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4696                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4697                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4698
4699                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4700                                                    ($drive->{mbps} || 0)*1024*1024,
4701                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4702                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4703                                                    $drive->{iops} || 0,
4704                                                    $drive->{iops_rd} || 0,
4705                                                    $drive->{iops_wr} || 0,
4706                                                    ($drive->{mbps_max} || 0)*1024*1024,
4707                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4708                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4709                                                    $drive->{iops_max} || 0,
4710                                                    $drive->{iops_rd_max} || 0,
4711                                                    $drive->{iops_wr_max} || 0,
4712                                                    $drive->{bps_max_length} || 1,
4713                                                    $drive->{bps_rd_max_length} || 1,
4714                                                    $drive->{bps_wr_max_length} || 1,
4715                                                    $drive->{iops_max_length} || 1,
4716                                                    $drive->{iops_rd_max_length} || 1,
4717                                                    $drive->{iops_wr_max_length} || 1);
4718
4719                     }
4720
4721                     return 1;
4722                 }
4723
4724             } else { # cdrom
4725
4726                 if ($drive->{file} eq 'none') {
4727                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4728                     if (drive_is_cloudinit($old_drive)) {
4729                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4730                     }
4731                 } else {
4732                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4733                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4734                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4735                 }
4736
4737                 return 1;
4738             }
4739         }
4740     }
4741
4742     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4743     # hotplug new disks
4744     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4745     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4746 }
4747
4748 sub vm_start {
4749     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4750         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4751
4752     PVE::QemuConfig->lock_config($vmid, sub {
4753         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4754
4755         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4756
4757         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4758
4759         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4760
4761         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4762             vmconfig_apply_pending($vmid, $conf, $storecfg);
4763             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4764         }
4765
4766         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4767
4768         my $defaults = load_defaults();
4769
4770         # set environment variable useful inside network script
4771         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4772
4773         my $local_volumes = {};
4774
4775         if ($targetstorage) {
4776             foreach_drive($conf, sub {
4777                 my ($ds, $drive) = @_;
4778
4779                 return if drive_is_cdrom($drive);
4780
4781                 my $volid = $drive->{file};
4782
4783                 return if !$volid;
4784
4785                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4786
4787                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4788                 return if $scfg->{shared};
4789                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4790             });
4791
4792             my $format = undef;
4793
4794             foreach my $opt (sort keys %$local_volumes) {
4795
4796                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4797                 my $drive = parse_drive($opt, $conf->{$opt});
4798
4799                 #if remote storage is specified, use default format
4800                 if ($targetstorage && $targetstorage ne "1") {
4801                     $storeid = $targetstorage;
4802                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4803                     $format = $defFormat;
4804                 } else {
4805                     #else we use same format than original
4806                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4807                     $format = qemu_img_format($scfg, $volid);
4808                 }
4809
4810                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4811                 my $newdrive = $drive;
4812                 $newdrive->{format} = $format;
4813                 $newdrive->{file} = $newvolid;
4814                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4815                 $local_volumes->{$opt} = $drivestr;
4816                 #pass drive to conf for command line
4817                 $conf->{$opt} = $drivestr;
4818             }
4819         }
4820
4821         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4822
4823         my $migrate_port = 0;
4824         my $migrate_uri;
4825         if ($statefile) {
4826             if ($statefile eq 'tcp') {
4827                 my $localip = "localhost";
4828                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4829                 my $nodename = PVE::INotify::nodename();
4830
4831                 if (!defined($migration_type)) {
4832                     if (defined($datacenterconf->{migration}->{type})) {
4833                         $migration_type = $datacenterconf->{migration}->{type};
4834                     } else {
4835                         $migration_type = 'secure';
4836                     }
4837                 }
4838
4839                 if ($migration_type eq 'insecure') {
4840                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4841                     if ($migrate_network_addr) {
4842                         $localip = $migrate_network_addr;
4843                     } else {
4844                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4845                     }
4846
4847                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4848                 }
4849
4850                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4851                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4852                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4853                 push @$cmd, '-incoming', $migrate_uri;
4854                 push @$cmd, '-S';
4855
4856             } elsif ($statefile eq 'unix') {
4857                 # should be default for secure migrations as a ssh TCP forward
4858                 # tunnel is not deterministic reliable ready and fails regurarly
4859                 # to set up in time, so use UNIX socket forwards
4860                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4861                 unlink $socket_addr;
4862
4863                 $migrate_uri = "unix:$socket_addr";
4864
4865                 push @$cmd, '-incoming', $migrate_uri;
4866                 push @$cmd, '-S';
4867
4868             } else {
4869                 push @$cmd, '-loadstate', $statefile;
4870             }
4871         } elsif ($paused) {
4872             push @$cmd, '-S';
4873         }
4874
4875         # host pci devices
4876         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4877           my $d = parse_hostpci($conf->{"hostpci$i"});
4878           next if !$d;
4879           my $pcidevices = $d->{pciid};
4880           foreach my $pcidevice (@$pcidevices) {
4881                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4882
4883                 my $info = pci_device_info("0000:$pciid");
4884                 die "IOMMU not present\n" if !check_iommu_support();
4885                 die "no pci device info for device '$pciid'\n" if !$info;
4886                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4887                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4888           }
4889         }
4890
4891         PVE::Storage::activate_volumes($storecfg, $vollist);
4892
4893         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4894             my $cmd = [];
4895             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4896             eval  { run_command($cmd); };
4897         }
4898
4899         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4900                                                   : $defaults->{cpuunits};
4901
4902         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4903         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4904
4905         my %properties = (
4906             Slice => 'qemu.slice',
4907             KillMode => 'none',
4908             CPUShares => $cpuunits
4909         );
4910
4911         if (my $cpulimit = $conf->{cpulimit}) {
4912             $properties{CPUQuota} = int($cpulimit * 100);
4913         }
4914         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4915
4916         if ($conf->{hugepages}) {
4917
4918             my $code = sub {
4919                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4920                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4921
4922                 PVE::QemuServer::Memory::hugepages_mount();
4923                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4924
4925                 eval  {
4926                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4927                     run_command($cmd, %run_params);
4928                 };
4929
4930                 if (my $err = $@) {
4931                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4932                     die $err;
4933                 }
4934
4935                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4936             };
4937             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4938
4939         } else {
4940             eval  {
4941                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4942                 run_command($cmd, %run_params);
4943             };
4944         }
4945
4946         if (my $err = $@) {
4947             # deactivate volumes if start fails
4948             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4949             die "start failed: $err";
4950         }
4951
4952         print "migration listens on $migrate_uri\n" if $migrate_uri;
4953
4954         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4955             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4956             warn $@ if $@;
4957         }
4958
4959         #start nbd server for storage migration
4960         if ($targetstorage) {
4961             my $nodename = PVE::INotify::nodename();
4962             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4963             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4964             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4965             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4966
4967             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4968
4969             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4970
4971             foreach my $opt (sort keys %$local_volumes) {
4972                 my $volid = $local_volumes->{$opt};
4973                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4974                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4975                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4976             }
4977         }
4978
4979         if ($migratedfrom) {
4980             eval {
4981                 set_migration_caps($vmid);
4982             };
4983             warn $@ if $@;
4984
4985             if ($spice_port) {
4986                 print "spice listens on port $spice_port\n";
4987                 if ($spice_ticket) {
4988                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4989                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4990                 }
4991             }
4992
4993         } else {
4994             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4995                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4996                     if $conf->{balloon};
4997             }
4998
4999             foreach my $opt (keys %$conf) {
5000                 next if $opt !~  m/^net\d+$/;
5001                 my $nicconf = parse_net($conf->{$opt});
5002                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5003             }
5004         }
5005
5006         vm_mon_cmd_nocheck($vmid, 'qom-set',
5007                     path => "machine/peripheral/balloon0",
5008                     property => "guest-stats-polling-interval",
5009                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5010
5011     });
5012 }
5013
5014 sub vm_mon_cmd {
5015     my ($vmid, $execute, %params) = @_;
5016
5017     my $cmd = { execute => $execute, arguments => \%params };
5018     vm_qmp_command($vmid, $cmd);
5019 }
5020
5021 sub vm_mon_cmd_nocheck {
5022     my ($vmid, $execute, %params) = @_;
5023
5024     my $cmd = { execute => $execute, arguments => \%params };
5025     vm_qmp_command($vmid, $cmd, 1);
5026 }
5027
5028 sub vm_qmp_command {
5029     my ($vmid, $cmd, $nocheck) = @_;
5030
5031     my $res;
5032
5033     my $timeout;
5034     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5035         $timeout = $cmd->{arguments}->{timeout};
5036         delete $cmd->{arguments}->{timeout};
5037     }
5038
5039     eval {
5040         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5041         my $sname = qmp_socket($vmid);
5042         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5043             my $qmpclient = PVE::QMPClient->new();
5044
5045             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5046         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
5047             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
5048                 if scalar(%{$cmd->{arguments}});
5049             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
5050         } else {
5051             die "unable to open monitor socket\n";
5052         }
5053     };
5054     if (my $err = $@) {
5055         syslog("err", "VM $vmid qmp command failed - $err");
5056         die $err;
5057     }
5058
5059     return $res;
5060 }
5061
5062 sub vm_human_monitor_command {
5063     my ($vmid, $cmdline) = @_;
5064
5065     my $res;
5066
5067     my $cmd = {
5068         execute => 'human-monitor-command',
5069         arguments => { 'command-line' => $cmdline},
5070     };
5071
5072     return vm_qmp_command($vmid, $cmd);
5073 }
5074
5075 sub vm_commandline {
5076     my ($storecfg, $vmid) = @_;
5077
5078     my $conf = PVE::QemuConfig->load_config($vmid);
5079
5080     my $defaults = load_defaults();
5081
5082     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5083
5084     return PVE::Tools::cmd2string($cmd);
5085 }
5086
5087 sub vm_reset {
5088     my ($vmid, $skiplock) = @_;
5089
5090     PVE::QemuConfig->lock_config($vmid, sub {
5091
5092         my $conf = PVE::QemuConfig->load_config($vmid);
5093
5094         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5095
5096         vm_mon_cmd($vmid, "system_reset");
5097     });
5098 }
5099
5100 sub get_vm_volumes {
5101     my ($conf) = @_;
5102
5103     my $vollist = [];
5104     foreach_volid($conf, sub {
5105         my ($volid, $attr) = @_;
5106
5107         return if $volid =~ m|^/|;
5108
5109         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5110         return if !$sid;
5111
5112         push @$vollist, $volid;
5113     });
5114
5115     return $vollist;
5116 }
5117
5118 sub vm_stop_cleanup {
5119     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5120
5121     eval {
5122
5123         if (!$keepActive) {
5124             my $vollist = get_vm_volumes($conf);
5125             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5126         }
5127
5128         foreach my $ext (qw(mon qmp pid vnc qga)) {
5129             unlink "/var/run/qemu-server/${vmid}.$ext";
5130         }
5131
5132         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5133     };
5134     warn $@ if $@; # avoid errors - just warn
5135 }
5136
5137 # Note: use $nockeck to skip tests if VM configuration file exists.
5138 # We need that when migration VMs to other nodes (files already moved)
5139 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5140 sub vm_stop {
5141     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5142
5143     $force = 1 if !defined($force) && !$shutdown;
5144
5145     if ($migratedfrom){
5146         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5147         kill 15, $pid if $pid;
5148         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5149         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5150         return;
5151     }
5152
5153     PVE::QemuConfig->lock_config($vmid, sub {
5154
5155         my $pid = check_running($vmid, $nocheck);
5156         return if !$pid;
5157
5158         my $conf;
5159         if (!$nocheck) {
5160             $conf = PVE::QemuConfig->load_config($vmid);
5161             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5162             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5163                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5164                 $timeout = $opts->{down} if $opts->{down};
5165             }
5166         }
5167
5168         $timeout = 60 if !defined($timeout);
5169
5170         eval {
5171             if ($shutdown) {
5172                 if (defined($conf) && $conf->{agent}) {
5173                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5174                 } else {
5175                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5176                 }
5177             } else {
5178                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5179             }
5180         };
5181         my $err = $@;
5182
5183         if (!$err) {
5184             my $count = 0;
5185             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5186                 $count++;
5187                 sleep 1;
5188             }
5189
5190             if ($count >= $timeout) {
5191                 if ($force) {
5192                     warn "VM still running - terminating now with SIGTERM\n";
5193                     kill 15, $pid;
5194                 } else {
5195                     die "VM quit/powerdown failed - got timeout\n";
5196                 }
5197             } else {
5198                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5199                 return;
5200             }
5201         } else {
5202             if ($force) {
5203                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5204                 kill 15, $pid;
5205             } else {
5206                 die "VM quit/powerdown failed\n";
5207             }
5208         }
5209
5210         # wait again
5211         $timeout = 10;
5212
5213         my $count = 0;
5214         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5215             $count++;
5216             sleep 1;
5217         }
5218
5219         if ($count >= $timeout) {
5220             warn "VM still running - terminating now with SIGKILL\n";
5221             kill 9, $pid;
5222             sleep 1;
5223         }
5224
5225         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5226    });
5227 }
5228
5229 sub vm_suspend {
5230     my ($vmid, $skiplock) = @_;
5231
5232     PVE::QemuConfig->lock_config($vmid, sub {
5233
5234         my $conf = PVE::QemuConfig->load_config($vmid);
5235
5236         PVE::QemuConfig->check_lock($conf)
5237             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5238
5239         vm_mon_cmd($vmid, "stop");
5240     });
5241 }
5242
5243 sub vm_resume {
5244     my ($vmid, $skiplock, $nocheck) = @_;
5245
5246     PVE::QemuConfig->lock_config($vmid, sub {
5247
5248         if (!$nocheck) {
5249
5250             my $conf = PVE::QemuConfig->load_config($vmid);
5251
5252             PVE::QemuConfig->check_lock($conf)
5253                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5254
5255             vm_mon_cmd($vmid, "cont");
5256
5257         } else {
5258             vm_mon_cmd_nocheck($vmid, "cont");
5259         }
5260     });
5261 }
5262
5263 sub vm_sendkey {
5264     my ($vmid, $skiplock, $key) = @_;
5265
5266     PVE::QemuConfig->lock_config($vmid, sub {
5267
5268         my $conf = PVE::QemuConfig->load_config($vmid);
5269
5270         # there is no qmp command, so we use the human monitor command
5271         vm_human_monitor_command($vmid, "sendkey $key");
5272     });
5273 }
5274
5275 sub vm_destroy {
5276     my ($storecfg, $vmid, $skiplock) = @_;
5277
5278     PVE::QemuConfig->lock_config($vmid, sub {
5279
5280         my $conf = PVE::QemuConfig->load_config($vmid);
5281
5282         if (!check_running($vmid)) {
5283             destroy_vm($storecfg, $vmid, undef, $skiplock);
5284         } else {
5285             die "VM $vmid is running - destroy failed\n";
5286         }
5287     });
5288 }
5289
5290 # pci helpers
5291
5292 sub file_write {
5293     my ($filename, $buf) = @_;
5294
5295     my $fh = IO::File->new($filename, "w");
5296     return undef if !$fh;
5297
5298     my $res = print $fh $buf;
5299
5300     $fh->close();
5301
5302     return $res;
5303 }
5304
5305 sub pci_device_info {
5306     my ($name) = @_;
5307
5308     my $res;
5309
5310     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5311     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5312
5313     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5314     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5315
5316     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5317     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5318
5319     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5320     return undef if !defined($product) || $product !~ s/^0x//;
5321
5322     $res = {
5323         name => $name,
5324         vendor => $vendor,
5325         product => $product,
5326         domain => $domain,
5327         bus => $bus,
5328         slot => $slot,
5329         func => $func,
5330         irq => $irq,
5331         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5332     };
5333
5334     return $res;
5335 }
5336
5337 sub pci_dev_reset {
5338     my ($dev) = @_;
5339
5340     my $name = $dev->{name};
5341
5342     my $fn = "$pcisysfs/devices/$name/reset";
5343
5344     return file_write($fn, "1");
5345 }
5346
5347 sub pci_dev_bind_to_vfio {
5348     my ($dev) = @_;
5349
5350     my $name = $dev->{name};
5351
5352     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5353
5354     if (!-d $vfio_basedir) {
5355         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5356     }
5357     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5358
5359     my $testdir = "$vfio_basedir/$name";
5360     return 1 if -d $testdir;
5361
5362     my $data = "$dev->{vendor} $dev->{product}";
5363     return undef if !file_write("$vfio_basedir/new_id", $data);
5364
5365     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5366     if (!file_write($fn, $name)) {
5367         return undef if -f $fn;
5368     }
5369
5370     $fn = "$vfio_basedir/bind";
5371     if (! -d $testdir) {
5372         return undef if !file_write($fn, $name);
5373     }
5374
5375     return -d $testdir;
5376 }
5377
5378 sub pci_dev_group_bind_to_vfio {
5379     my ($pciid) = @_;
5380
5381     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5382
5383     if (!-d $vfio_basedir) {
5384         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5385     }
5386     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5387
5388     # get IOMMU group devices
5389     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5390       my @devs = grep /^0000:/, readdir($D);
5391     closedir($D);
5392
5393     foreach my $pciid (@devs) {
5394         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5395
5396         # pci bridges, switches or root ports are not supported
5397         # they have a pci_bus subdirectory so skip them
5398         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5399
5400         my $info = pci_device_info($1);
5401         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5402     }
5403
5404     return 1;
5405 }
5406
5407 # vzdump restore implementaion
5408
5409 sub tar_archive_read_firstfile {
5410     my $archive = shift;
5411
5412     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5413
5414     # try to detect archive type first
5415     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5416         die "unable to open file '$archive'\n";
5417     my $firstfile = <$fh>;
5418     kill 15, $pid;
5419     close $fh;
5420
5421     die "ERROR: archive contaions no data\n" if !$firstfile;
5422     chomp $firstfile;
5423
5424     return $firstfile;
5425 }
5426
5427 sub tar_restore_cleanup {
5428     my ($storecfg, $statfile) = @_;
5429
5430     print STDERR "starting cleanup\n";
5431
5432     if (my $fd = IO::File->new($statfile, "r")) {
5433         while (defined(my $line = <$fd>)) {
5434             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5435                 my $volid = $2;
5436                 eval {
5437                     if ($volid =~ m|^/|) {
5438                         unlink $volid || die 'unlink failed\n';
5439                     } else {
5440                         PVE::Storage::vdisk_free($storecfg, $volid);
5441                     }
5442                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5443                 };
5444                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5445             } else {
5446                 print STDERR "unable to parse line in statfile - $line";
5447             }
5448         }
5449         $fd->close();
5450     }
5451 }
5452
5453 sub restore_archive {
5454     my ($archive, $vmid, $user, $opts) = @_;
5455
5456     my $format = $opts->{format};
5457     my $comp;
5458
5459     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5460         $format = 'tar' if !$format;
5461         $comp = 'gzip';
5462     } elsif ($archive =~ m/\.tar$/) {
5463         $format = 'tar' if !$format;
5464     } elsif ($archive =~ m/.tar.lzo$/) {
5465         $format = 'tar' if !$format;
5466         $comp = 'lzop';
5467     } elsif ($archive =~ m/\.vma$/) {
5468         $format = 'vma' if !$format;
5469     } elsif ($archive =~ m/\.vma\.gz$/) {
5470         $format = 'vma' if !$format;
5471         $comp = 'gzip';
5472     } elsif ($archive =~ m/\.vma\.lzo$/) {
5473         $format = 'vma' if !$format;
5474         $comp = 'lzop';
5475     } else {
5476         $format = 'vma' if !$format; # default
5477     }
5478
5479     # try to detect archive format
5480     if ($format eq 'tar') {
5481         return restore_tar_archive($archive, $vmid, $user, $opts);
5482     } else {
5483         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5484     }
5485 }
5486
5487 sub restore_update_config_line {
5488     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5489
5490     return if $line =~ m/^\#qmdump\#/;
5491     return if $line =~ m/^\#vzdump\#/;
5492     return if $line =~ m/^lock:/;
5493     return if $line =~ m/^unused\d+:/;
5494     return if $line =~ m/^parent:/;
5495     return if $line =~ m/^template:/; # restored VM is never a template
5496
5497     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5498     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5499         # try to convert old 1.X settings
5500         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5501         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5502             my ($model, $macaddr) = split(/\=/, $devconfig);
5503             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5504             my $net = {
5505                 model => $model,
5506                 bridge => "vmbr$ind",
5507                 macaddr => $macaddr,
5508             };
5509             my $netstr = print_net($net);
5510
5511             print $outfd "net$cookie->{netcount}: $netstr\n";
5512             $cookie->{netcount}++;
5513         }
5514     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5515         my ($id, $netstr) = ($1, $2);
5516         my $net = parse_net($netstr);
5517         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5518         $netstr = print_net($net);
5519         print $outfd "$id: $netstr\n";
5520     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5521         my $virtdev = $1;
5522         my $value = $3;
5523         my $di = parse_drive($virtdev, $value);
5524         if (defined($di->{backup}) && !$di->{backup}) {
5525             print $outfd "#$line";
5526         } elsif ($map->{$virtdev}) {
5527             delete $di->{format}; # format can change on restore
5528             $di->{file} = $map->{$virtdev};
5529             $value = print_drive($vmid, $di);
5530             print $outfd "$virtdev: $value\n";
5531         } else {
5532             print $outfd $line;
5533         }
5534     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5535         my ($uuid, $uuid_str);
5536         UUID::generate($uuid);
5537         UUID::unparse($uuid, $uuid_str);
5538         my $smbios1 = parse_smbios1($2);
5539         $smbios1->{uuid} = $uuid_str;
5540         print $outfd $1.print_smbios1($smbios1)."\n";
5541     } else {
5542         print $outfd $line;
5543     }
5544 }
5545
5546 sub scan_volids {
5547     my ($cfg, $vmid) = @_;
5548
5549     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5550
5551     my $volid_hash = {};
5552     foreach my $storeid (keys %$info) {
5553         foreach my $item (@{$info->{$storeid}}) {
5554             next if !($item->{volid} && $item->{size});
5555             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5556             $volid_hash->{$item->{volid}} = $item;
5557         }
5558     }
5559
5560     return $volid_hash;
5561 }
5562
5563 sub is_volume_in_use {
5564     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5565
5566     my $path = PVE::Storage::path($storecfg, $volid);
5567
5568     my $scan_config = sub {
5569         my ($cref, $snapname) = @_;
5570
5571         foreach my $key (keys %$cref) {
5572             my $value = $cref->{$key};
5573             if (is_valid_drivename($key)) {
5574                 next if $skip_drive && $key eq $skip_drive;
5575                 my $drive = parse_drive($key, $value);
5576                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5577                 return 1 if $volid eq $drive->{file};
5578                 if ($drive->{file} =~ m!^/!) {
5579                     return 1 if $drive->{file} eq $path;
5580                 } else {
5581                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5582                     next if !$storeid;
5583                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5584                     next if !$scfg;
5585                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5586                 }
5587             }
5588         }
5589
5590         return 0;
5591     };
5592
5593     return 1 if &$scan_config($conf);
5594
5595     undef $skip_drive;
5596
5597     foreach my $snapname (keys %{$conf->{snapshots}}) {
5598         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5599     }
5600
5601     return 0;
5602 }
5603
5604 sub update_disksize {
5605     my ($vmid, $conf, $volid_hash) = @_;
5606
5607     my $changes;
5608
5609     # used and unused disks
5610     my $referenced = {};
5611
5612     # Note: it is allowed to define multiple storages with same path (alias), so
5613     # we need to check both 'volid' and real 'path' (two different volid can point
5614     # to the same path).
5615
5616     my $referencedpath = {};
5617
5618     # update size info
5619     foreach my $opt (keys %$conf) {
5620         if (is_valid_drivename($opt)) {
5621             my $drive = parse_drive($opt, $conf->{$opt});
5622             my $volid = $drive->{file};
5623             next if !$volid;
5624
5625             $referenced->{$volid} = 1;
5626             if ($volid_hash->{$volid} &&
5627                 (my $path = $volid_hash->{$volid}->{path})) {
5628                 $referencedpath->{$path} = 1;
5629             }
5630
5631             next if drive_is_cdrom($drive);
5632             next if !$volid_hash->{$volid};
5633
5634             $drive->{size} = $volid_hash->{$volid}->{size};
5635             my $new = print_drive($vmid, $drive);
5636             if ($new ne $conf->{$opt}) {
5637                 $changes = 1;
5638                 $conf->{$opt} = $new;
5639             }
5640         }
5641     }
5642
5643     # remove 'unusedX' entry if volume is used
5644     foreach my $opt (keys %$conf) {
5645         next if $opt !~ m/^unused\d+$/;
5646         my $volid = $conf->{$opt};
5647         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5648         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5649             $changes = 1;
5650             delete $conf->{$opt};
5651         }
5652
5653         $referenced->{$volid} = 1;
5654         $referencedpath->{$path} = 1 if $path;
5655     }
5656
5657     foreach my $volid (sort keys %$volid_hash) {
5658         next if $volid =~ m/vm-$vmid-state-/;
5659         next if $referenced->{$volid};
5660         my $path = $volid_hash->{$volid}->{path};
5661         next if !$path; # just to be sure
5662         next if $referencedpath->{$path};
5663         $changes = 1;
5664         PVE::QemuConfig->add_unused_volume($conf, $volid);
5665         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5666     }
5667
5668     return $changes;
5669 }
5670
5671 sub rescan {
5672     my ($vmid, $nolock) = @_;
5673
5674     my $cfg = PVE::Storage::config();
5675
5676     my $volid_hash = scan_volids($cfg, $vmid);
5677
5678     my $updatefn =  sub {
5679         my ($vmid) = @_;
5680
5681         my $conf = PVE::QemuConfig->load_config($vmid);
5682
5683         PVE::QemuConfig->check_lock($conf);
5684
5685         my $vm_volids = {};
5686         foreach my $volid (keys %$volid_hash) {
5687             my $info = $volid_hash->{$volid};
5688             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5689         }
5690
5691         my $changes = update_disksize($vmid, $conf, $vm_volids);
5692
5693         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5694     };
5695
5696     if (defined($vmid)) {
5697         if ($nolock) {
5698             &$updatefn($vmid);
5699         } else {
5700             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5701         }
5702     } else {
5703         my $vmlist = config_list();
5704         foreach my $vmid (keys %$vmlist) {
5705             if ($nolock) {
5706                 &$updatefn($vmid);
5707             } else {
5708                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5709             }
5710         }
5711     }
5712 }
5713
5714 sub restore_vma_archive {
5715     my ($archive, $vmid, $user, $opts, $comp) = @_;
5716
5717     my $input = $archive eq '-' ? "<&STDIN" : undef;
5718     my $readfrom = $archive;
5719
5720     my $uncomp = '';
5721     if ($comp) {
5722         $readfrom = '-';
5723         my $qarchive = PVE::Tools::shellquote($archive);
5724         if ($comp eq 'gzip') {
5725             $uncomp = "zcat $qarchive|";
5726         } elsif ($comp eq 'lzop') {
5727             $uncomp = "lzop -d -c $qarchive|";
5728         } else {
5729             die "unknown compression method '$comp'\n";
5730         }
5731
5732     }
5733
5734     my $tmpdir = "/var/tmp/vzdumptmp$$";
5735     rmtree $tmpdir;
5736
5737     # disable interrupts (always do cleanups)
5738     local $SIG{INT} =
5739         local $SIG{TERM} =
5740         local $SIG{QUIT} =
5741         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5742
5743     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5744     POSIX::mkfifo($mapfifo, 0600);
5745     my $fifofh;
5746
5747     my $openfifo = sub {
5748         open($fifofh, '>', $mapfifo) || die $!;
5749     };
5750
5751     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5752
5753     my $oldtimeout;
5754     my $timeout = 5;
5755
5756     my $devinfo = {};
5757
5758     my $rpcenv = PVE::RPCEnvironment::get();
5759
5760     my $conffile = PVE::QemuConfig->config_file($vmid);
5761     my $tmpfn = "$conffile.$$.tmp";
5762
5763     # Note: $oldconf is undef if VM does not exists
5764     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5765     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5766
5767     my $print_devmap = sub {
5768         my $virtdev_hash = {};
5769
5770         my $cfgfn = "$tmpdir/qemu-server.conf";
5771
5772         # we can read the config - that is already extracted
5773         my $fh = IO::File->new($cfgfn, "r") ||
5774             "unable to read qemu-server.conf - $!\n";
5775
5776         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5777         if (-f $fwcfgfn) {
5778             my $pve_firewall_dir = '/etc/pve/firewall';
5779             mkdir $pve_firewall_dir; # make sure the dir exists
5780             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5781         }
5782
5783         while (defined(my $line = <$fh>)) {
5784             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5785                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5786                 die "archive does not contain data for drive '$virtdev'\n"
5787                     if !$devinfo->{$devname};
5788                 if (defined($opts->{storage})) {
5789                     $storeid = $opts->{storage} || 'local';
5790                 } elsif (!$storeid) {
5791                     $storeid = 'local';
5792                 }
5793                 $format = 'raw' if !$format;
5794                 $devinfo->{$devname}->{devname} = $devname;
5795                 $devinfo->{$devname}->{virtdev} = $virtdev;
5796                 $devinfo->{$devname}->{format} = $format;
5797                 $devinfo->{$devname}->{storeid} = $storeid;
5798
5799                 # check permission on storage
5800                 my $pool = $opts->{pool}; # todo: do we need that?
5801                 if ($user ne 'root@pam') {
5802                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5803                 }
5804
5805                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5806             }
5807         }
5808
5809         foreach my $devname (keys %$devinfo) {
5810             die "found no device mapping information for device '$devname'\n"
5811                 if !$devinfo->{$devname}->{virtdev};
5812         }
5813
5814         my $cfg = PVE::Storage::config();
5815
5816         # create empty/temp config
5817         if ($oldconf) {
5818             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5819             foreach_drive($oldconf, sub {
5820                 my ($ds, $drive) = @_;
5821
5822                 return if drive_is_cdrom($drive);
5823
5824                 my $volid = $drive->{file};
5825
5826                 return if !$volid || $volid =~ m|^/|;
5827
5828                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5829                 return if !$path || !$owner || ($owner != $vmid);
5830
5831                 # Note: only delete disk we want to restore
5832                 # other volumes will become unused
5833                 if ($virtdev_hash->{$ds}) {
5834                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5835                     if (my $err = $@) {
5836                         warn $err;
5837                     }
5838                 }
5839             });
5840
5841             # delete vmstate files
5842             # since after the restore we have no snapshots anymore
5843             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5844                 my $snap = $oldconf->{snapshots}->{$snapname};
5845                 if ($snap->{vmstate}) {
5846                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5847                     if (my $err = $@) {
5848                         warn $err;
5849                     }
5850                 }
5851             }
5852         }
5853
5854         my $map = {};
5855         foreach my $virtdev (sort keys %$virtdev_hash) {
5856             my $d = $virtdev_hash->{$virtdev};
5857             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5858             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5859
5860             # test if requested format is supported
5861             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5862             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5863             $d->{format} = $defFormat if !$supported;
5864
5865             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5866                                                   $d->{format}, undef, $alloc_size);
5867             print STDERR "new volume ID is '$volid'\n";
5868             $d->{volid} = $volid;
5869             my $path = PVE::Storage::path($cfg, $volid);
5870
5871             PVE::Storage::activate_volumes($cfg,[$volid]);
5872
5873             my $write_zeros = 1;
5874             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5875                 $write_zeros = 0;
5876             }
5877
5878             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5879
5880             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5881             $map->{$virtdev} = $volid;
5882         }
5883
5884         $fh->seek(0, 0) || die "seek failed - $!\n";
5885
5886         my $outfd = new IO::File ($tmpfn, "w") ||
5887             die "unable to write config for VM $vmid\n";
5888
5889         my $cookie = { netcount => 0 };
5890         while (defined(my $line = <$fh>)) {
5891             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5892         }
5893
5894         $fh->close();
5895         $outfd->close();
5896     };
5897
5898     eval {
5899         # enable interrupts
5900         local $SIG{INT} =
5901             local $SIG{TERM} =
5902             local $SIG{QUIT} =
5903             local $SIG{HUP} =
5904             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5905         local $SIG{ALRM} = sub { die "got timeout\n"; };
5906
5907         $oldtimeout = alarm($timeout);
5908
5909         my $parser = sub {
5910             my $line = shift;
5911
5912             print "$line\n";
5913
5914             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5915                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5916                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5917             } elsif ($line =~ m/^CTIME: /) {
5918                 # we correctly received the vma config, so we can disable
5919                 # the timeout now for disk allocation (set to 10 minutes, so
5920                 # that we always timeout if something goes wrong)
5921                 alarm(600);
5922                 &$print_devmap();
5923                 print $fifofh "done\n";
5924                 my $tmp = $oldtimeout || 0;
5925                 $oldtimeout = undef;
5926                 alarm($tmp);
5927                 close($fifofh);
5928             }
5929         };
5930
5931         print "restore vma archive: $cmd\n";
5932         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5933     };
5934     my $err = $@;
5935
5936     alarm($oldtimeout) if $oldtimeout;
5937
5938     my $vollist = [];
5939     foreach my $devname (keys %$devinfo) {
5940         my $volid = $devinfo->{$devname}->{volid};
5941         push @$vollist, $volid if $volid;
5942     }
5943
5944     my $cfg = PVE::Storage::config();
5945     PVE::Storage::deactivate_volumes($cfg, $vollist);
5946
5947     unlink $mapfifo;
5948
5949     if ($err) {
5950         rmtree $tmpdir;
5951         unlink $tmpfn;
5952
5953         foreach my $devname (keys %$devinfo) {
5954             my $volid = $devinfo->{$devname}->{volid};
5955             next if !$volid;
5956             eval {
5957                 if ($volid =~ m|^/|) {
5958                     unlink $volid || die 'unlink failed\n';
5959                 } else {
5960                     PVE::Storage::vdisk_free($cfg, $volid);
5961                 }
5962                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5963             };
5964             print STDERR "unable to cleanup '$volid' - $@" if $@;
5965         }
5966         die $err;
5967     }
5968
5969     rmtree $tmpdir;
5970
5971     rename($tmpfn, $conffile) ||
5972         die "unable to commit configuration file '$conffile'\n";
5973
5974     PVE::Cluster::cfs_update(); # make sure we read new file
5975
5976     eval { rescan($vmid, 1); };
5977     warn $@ if $@;
5978 }
5979
5980 sub restore_tar_archive {
5981     my ($archive, $vmid, $user, $opts) = @_;
5982
5983     if ($archive ne '-') {
5984         my $firstfile = tar_archive_read_firstfile($archive);
5985         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5986             if $firstfile ne 'qemu-server.conf';
5987     }
5988
5989     my $storecfg = PVE::Storage::config();
5990
5991     # destroy existing data - keep empty config
5992     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5993     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5994
5995     my $tocmd = "/usr/lib/qemu-server/qmextract";
5996
5997     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5998     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5999     $tocmd .= ' --prealloc' if $opts->{prealloc};
6000     $tocmd .= ' --info' if $opts->{info};
6001
6002     # tar option "xf" does not autodetect compression when read from STDIN,
6003     # so we pipe to zcat
6004     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6005         PVE::Tools::shellquote("--to-command=$tocmd");
6006
6007     my $tmpdir = "/var/tmp/vzdumptmp$$";
6008     mkpath $tmpdir;
6009
6010     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6011     local $ENV{VZDUMP_VMID} = $vmid;
6012     local $ENV{VZDUMP_USER} = $user;
6013
6014     my $conffile = PVE::QemuConfig->config_file($vmid);
6015     my $tmpfn = "$conffile.$$.tmp";
6016
6017     # disable interrupts (always do cleanups)
6018     local $SIG{INT} =
6019         local $SIG{TERM} =
6020         local $SIG{QUIT} =
6021         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6022
6023     eval {
6024         # enable interrupts
6025         local $SIG{INT} =
6026             local $SIG{TERM} =
6027             local $SIG{QUIT} =
6028             local $SIG{HUP} =
6029             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6030
6031         if ($archive eq '-') {
6032             print "extracting archive from STDIN\n";
6033             run_command($cmd, input => "<&STDIN");
6034         } else {
6035             print "extracting archive '$archive'\n";
6036             run_command($cmd);
6037         }
6038
6039         return if $opts->{info};
6040
6041         # read new mapping
6042         my $map = {};
6043         my $statfile = "$tmpdir/qmrestore.stat";
6044         if (my $fd = IO::File->new($statfile, "r")) {
6045             while (defined (my $line = <$fd>)) {
6046                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6047                     $map->{$1} = $2 if $1;
6048                 } else {
6049                     print STDERR "unable to parse line in statfile - $line\n";
6050                 }
6051             }
6052             $fd->close();
6053         }
6054
6055         my $confsrc = "$tmpdir/qemu-server.conf";
6056
6057         my $srcfd = new IO::File($confsrc, "r") ||
6058             die "unable to open file '$confsrc'\n";
6059
6060         my $outfd = new IO::File ($tmpfn, "w") ||
6061             die "unable to write config for VM $vmid\n";
6062
6063         my $cookie = { netcount => 0 };
6064         while (defined (my $line = <$srcfd>)) {
6065             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6066         }
6067
6068         $srcfd->close();
6069         $outfd->close();
6070     };
6071     my $err = $@;
6072
6073     if ($err) {
6074
6075         unlink $tmpfn;
6076
6077         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6078
6079         die $err;
6080     }
6081
6082     rmtree $tmpdir;
6083
6084     rename $tmpfn, $conffile ||
6085         die "unable to commit configuration file '$conffile'\n";
6086
6087     PVE::Cluster::cfs_update(); # make sure we read new file
6088
6089     eval { rescan($vmid, 1); };
6090     warn $@ if $@;
6091 };
6092
6093 sub foreach_storage_used_by_vm {
6094     my ($conf, $func) = @_;
6095
6096     my $sidhash = {};
6097
6098     foreach_drive($conf, sub {
6099         my ($ds, $drive) = @_;
6100         return if drive_is_cdrom($drive);
6101
6102         my $volid = $drive->{file};
6103
6104         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6105         $sidhash->{$sid} = $sid if $sid;
6106     });
6107
6108     foreach my $sid (sort keys %$sidhash) {
6109         &$func($sid);
6110     }
6111 }
6112
6113 sub do_snapshots_with_qemu {
6114     my ($storecfg, $volid) = @_;
6115
6116     my $storage_name = PVE::Storage::parse_volume_id($volid);
6117
6118     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6119         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6120         return 1;
6121     }
6122
6123     if ($volid =~ m/\.(qcow2|qed)$/){
6124         return 1;
6125     }
6126
6127     return undef;
6128 }
6129
6130 sub qga_check_running {
6131     my ($vmid) = @_;
6132
6133     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6134     if ($@) {
6135         warn "Qemu Guest Agent is not running - $@";
6136         return 0;
6137     }
6138     return 1;
6139 }
6140
6141 sub template_create {
6142     my ($vmid, $conf, $disk) = @_;
6143
6144     my $storecfg = PVE::Storage::config();
6145
6146     foreach_drive($conf, sub {
6147         my ($ds, $drive) = @_;
6148
6149         return if drive_is_cdrom($drive);
6150         return if $disk && $ds ne $disk;
6151
6152         my $volid = $drive->{file};
6153         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6154
6155         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6156         $drive->{file} = $voliddst;
6157         $conf->{$ds} = print_drive($vmid, $drive);
6158         PVE::QemuConfig->write_config($vmid, $conf);
6159     });
6160 }
6161
6162 sub qemu_img_convert {
6163     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6164
6165     my $storecfg = PVE::Storage::config();
6166     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6167     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6168
6169     if ($src_storeid && $dst_storeid) {
6170
6171         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6172
6173         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6174         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6175
6176         my $src_format = qemu_img_format($src_scfg, $src_volname);
6177         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6178
6179         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6180         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6181
6182         my $cmd = [];
6183         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6184         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6185         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6186         if ($is_zero_initialized) {
6187             push @$cmd, "zeroinit:$dst_path";
6188         } else {
6189             push @$cmd, $dst_path;
6190         }
6191
6192         my $parser = sub {
6193             my $line = shift;
6194             if($line =~ m/\((\S+)\/100\%\)/){
6195                 my $percent = $1;
6196                 my $transferred = int($size * $percent / 100);
6197                 my $remaining = $size - $transferred;
6198
6199                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6200             }
6201
6202         };
6203
6204         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6205         my $err = $@;
6206         die "copy failed: $err" if $err;
6207     }
6208 }
6209
6210 sub qemu_img_format {
6211     my ($scfg, $volname) = @_;
6212
6213     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6214         return $1;
6215     } else {
6216         return "raw";
6217     }
6218 }
6219
6220 sub qemu_drive_mirror {
6221     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6222
6223     $jobs = {} if !$jobs;
6224
6225     my $qemu_target;
6226     my $format;
6227     $jobs->{"drive-$drive"} = {};
6228
6229     if ($dst_volid =~ /^nbd:/) {
6230         $qemu_target = $dst_volid;
6231         $format = "nbd";
6232     } else {
6233         my $storecfg = PVE::Storage::config();
6234         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6235
6236         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6237
6238         $format = qemu_img_format($dst_scfg, $dst_volname);
6239
6240         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6241
6242         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6243     }
6244
6245     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6246     $opts->{format} = $format if $format;
6247
6248     print "drive mirror is starting for drive-$drive\n";
6249
6250     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6251
6252     if (my $err = $@) {
6253         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6254         die "mirroring error: $err";
6255     }
6256
6257     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6258 }
6259
6260 sub qemu_drive_mirror_monitor {
6261     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6262
6263     eval {
6264         my $err_complete = 0;
6265
6266         while (1) {
6267             die "storage migration timed out\n" if $err_complete > 300;
6268
6269             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6270
6271             my $running_mirror_jobs = {};
6272             foreach my $stat (@$stats) {
6273                 next if $stat->{type} ne 'mirror';
6274                 $running_mirror_jobs->{$stat->{device}} = $stat;
6275             }
6276
6277             my $readycounter = 0;
6278
6279             foreach my $job (keys %$jobs) {
6280
6281                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6282                     print "$job : finished\n";
6283                     delete $jobs->{$job};
6284                     next;
6285                 }
6286
6287                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6288
6289                 my $busy = $running_mirror_jobs->{$job}->{busy};
6290                 my $ready = $running_mirror_jobs->{$job}->{ready};
6291                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6292                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6293                     my $remaining = $total - $transferred;
6294                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6295
6296                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6297                 }
6298
6299                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6300             }
6301
6302             last if scalar(keys %$jobs) == 0;
6303
6304             if ($readycounter == scalar(keys %$jobs)) {
6305                 print "all mirroring jobs are ready \n";
6306                 last if $skipcomplete; #do the complete later
6307
6308                 if ($vmiddst && $vmiddst != $vmid) {
6309                     my $agent_running = $qga && qga_check_running($vmid);
6310                     if ($agent_running) {
6311                         print "freeze filesystem\n";
6312                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6313                     } else {
6314                         print "suspend vm\n";
6315                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6316                     }
6317
6318                     # if we clone a disk for a new target vm, we don't switch the disk
6319                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6320
6321                     if ($agent_running) {
6322                         print "unfreeze filesystem\n";
6323                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6324                     } else {
6325                         print "resume vm\n";
6326                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6327                     }
6328
6329                     last;
6330                 } else {
6331
6332                     foreach my $job (keys %$jobs) {
6333                         # try to switch the disk if source and destination are on the same guest
6334                         print "$job: Completing block job...\n";
6335
6336                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6337                         if ($@ =~ m/cannot be completed/) {
6338                             print "$job: Block job cannot be completed, try again.\n";
6339                             $err_complete++;
6340                         }else {
6341                             print "$job: Completed successfully.\n";
6342                             $jobs->{$job}->{complete} = 1;
6343                         }
6344                     }
6345                 }
6346             }
6347             sleep 1;
6348         }
6349     };
6350     my $err = $@;
6351
6352     if ($err) {
6353         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6354         die "mirroring error: $err";
6355     }
6356
6357 }
6358
6359 sub qemu_blockjobs_cancel {
6360     my ($vmid, $jobs) = @_;
6361
6362     foreach my $job (keys %$jobs) {
6363         print "$job: Cancelling block job\n";
6364         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6365         $jobs->{$job}->{cancel} = 1;
6366     }
6367
6368     while (1) {
6369         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6370
6371         my $running_jobs = {};
6372         foreach my $stat (@$stats) {
6373             $running_jobs->{$stat->{device}} = $stat;
6374         }
6375
6376         foreach my $job (keys %$jobs) {
6377
6378             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6379                 print "$job: Done.\n";
6380                 delete $jobs->{$job};
6381             }
6382         }
6383
6384         last if scalar(keys %$jobs) == 0;
6385
6386         sleep 1;
6387     }
6388 }
6389
6390 sub clone_disk {
6391     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6392         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6393
6394     my $newvolid;
6395
6396     if (!$full) {
6397         print "create linked clone of drive $drivename ($drive->{file})\n";
6398         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6399         push @$newvollist, $newvolid;
6400     } else {
6401
6402         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6403         $storeid = $storage if $storage;
6404
6405         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6406         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6407
6408         print "create full clone of drive $drivename ($drive->{file})\n";
6409         my $name = undef;
6410         if (drive_is_cloudinit($drive)) {
6411             $name = "vm-$newvmid-cloudinit";
6412             # cloudinit only supports raw and qcow2 atm:
6413             if ($dst_format eq 'qcow2') {
6414                 $name .= '.qcow2';
6415             } elsif ($dst_format ne 'raw') {
6416                 die "clone: unhandled format for cloudinit image\n";
6417             }
6418         }
6419         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6420         push @$newvollist, $newvolid;
6421
6422         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6423
6424         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6425         if (!$running || $snapname) {
6426             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6427         } else {
6428
6429             my $kvmver = get_running_qemu_version ($vmid);
6430             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6431                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6432                     if $drive->{iothread};
6433             }
6434
6435             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6436         }
6437     }
6438
6439     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6440
6441     my $disk = $drive;
6442     $disk->{format} = undef;
6443     $disk->{file} = $newvolid;
6444     $disk->{size} = $size;
6445
6446     return $disk;
6447 }
6448
6449 # this only works if VM is running
6450 sub get_current_qemu_machine {
6451     my ($vmid) = @_;
6452
6453     my $cmd = { execute => 'query-machines', arguments => {} };
6454     my $res = vm_qmp_command($vmid, $cmd);
6455
6456     my ($current, $default);
6457     foreach my $e (@$res) {
6458         $default = $e->{name} if $e->{'is-default'};
6459         $current = $e->{name} if $e->{'is-current'};
6460     }
6461
6462     # fallback to the default machine if current is not supported by qemu
6463     return $current || $default || 'pc';
6464 }
6465
6466 sub get_running_qemu_version {
6467     my ($vmid) = @_;
6468     my $cmd = { execute => 'query-version', arguments => {} };
6469     my $res = vm_qmp_command($vmid, $cmd);
6470     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6471 }
6472
6473 sub qemu_machine_feature_enabled {
6474     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6475
6476     my $current_major;
6477     my $current_minor;
6478
6479     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6480
6481         $current_major = $3;
6482         $current_minor = $4;
6483
6484     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6485
6486         $current_major = $1;
6487         $current_minor = $2;
6488     }
6489
6490     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6491
6492
6493 }
6494
6495 sub qemu_machine_pxe {
6496     my ($vmid, $conf, $machine) = @_;
6497
6498     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6499
6500     foreach my $opt (keys %$conf) {
6501         next if $opt !~ m/^net(\d+)$/;
6502         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6503         next if !$net;
6504         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6505         return $machine.".pxe" if $romfile =~ m/pxe/;
6506         last;
6507     }
6508
6509     return $machine;
6510 }
6511
6512 sub qemu_use_old_bios_files {
6513     my ($machine_type) = @_;
6514
6515     return if !$machine_type;
6516
6517     my $use_old_bios_files = undef;
6518
6519     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6520         $machine_type = $1;
6521         $use_old_bios_files = 1;
6522     } else {
6523         my $kvmver = kvm_user_version();
6524         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6525         # load new efi bios files on migration. So this hack is required to allow
6526         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6527         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6528         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6529     }
6530
6531     return ($use_old_bios_files, $machine_type);
6532 }
6533
6534 sub create_efidisk {
6535     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6536
6537     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6538
6539     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6540     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6541     PVE::Storage::activate_volumes($storecfg, [$volid]);
6542
6543     my $path = PVE::Storage::path($storecfg, $volid);
6544     eval {
6545         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6546     };
6547     die "Copying EFI vars image failed: $@" if $@;
6548
6549     return ($volid, $vars_size);
6550 }
6551
6552 sub lspci {
6553
6554     my $devices = {};
6555
6556     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6557             my (undef, $id, $function) = @_;
6558             my $res = { id => $id, function => $function};
6559             push @{$devices->{$id}}, $res;
6560     });
6561
6562     # Entries should be sorted by functions.
6563     foreach my $id (keys %$devices) {
6564         my $dev = $devices->{$id};
6565         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6566     }
6567
6568     return $devices;
6569 }
6570
6571 sub vm_iothreads_list {
6572     my ($vmid) = @_;
6573
6574     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6575
6576     my $iothreads = {};
6577     foreach my $iothread (@$res) {
6578         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6579     }
6580
6581     return $iothreads;
6582 }
6583
6584 sub scsihw_infos {
6585     my ($conf, $drive) = @_;
6586
6587     my $maxdev = 0;
6588
6589     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6590         $maxdev = 7;
6591     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6592         $maxdev = 1;
6593     } else {
6594         $maxdev = 256;
6595     }
6596
6597     my $controller = int($drive->{index} / $maxdev);
6598     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6599
6600     return ($maxdev, $controller, $controller_prefix);
6601 }
6602
6603 sub add_hyperv_enlightenments {
6604     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6605
6606     return if $winversion < 6;
6607     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6608
6609     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6610
6611     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6612         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6613         push @$cpuFlags , 'hv_vapic';
6614         push @$cpuFlags , 'hv_time';
6615     } else {
6616         push @$cpuFlags , 'hv_spinlocks=0xffff';
6617     }
6618
6619     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6620         push @$cpuFlags , 'hv_reset';
6621         push @$cpuFlags , 'hv_vpindex';
6622         push @$cpuFlags , 'hv_runtime';
6623     }
6624
6625     if ($winversion >= 7) {
6626         push @$cpuFlags , 'hv_relaxed';
6627     }
6628 }
6629
6630 sub windows_version {
6631     my ($ostype) = @_;
6632
6633     return 0 if !$ostype;
6634
6635     my $winversion = 0;
6636
6637     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6638         $winversion = 5;
6639     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6640         $winversion = 6;
6641     } elsif ($ostype =~ m/^win(\d+)$/) {
6642         $winversion = $1;
6643     }
6644
6645     return $winversion;
6646 }
6647
6648 sub resolve_dst_disk_format {
6649         my ($storecfg, $storeid, $src_volname, $format) = @_;
6650         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6651
6652         if (!$format) {
6653             # if no target format is specified, use the source disk format as hint
6654             if ($src_volname) {
6655                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6656                 $format = qemu_img_format($scfg, $src_volname);
6657             } else {
6658                 return $defFormat;
6659             }
6660         }
6661
6662         # test if requested format is supported - else use default
6663         my $supported = grep { $_ eq $format } @$validFormats;
6664         $format = $defFormat if !$supported;
6665         return $format;
6666 }
6667
6668 sub resolve_first_disk {
6669     my $conf = shift;
6670     my @disks = PVE::QemuServer::valid_drive_names();
6671     my $firstdisk;
6672     foreach my $ds (reverse @disks) {
6673         next if !$conf->{$ds};
6674         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6675         next if PVE::QemuServer::drive_is_cdrom($disk);
6676         $firstdisk = $ds;
6677     }
6678     return $firstdisk;
6679 }
6680
6681 sub generate_smbios1_uuid {
6682     my ($uuid, $uuid_str);
6683     UUID::generate($uuid);
6684     UUID::unparse($uuid, $uuid_str);
6685     return "uuid=$uuid_str";
6686 }
6687
6688 # bash completion helper
6689
6690 sub complete_backup_archives {
6691     my ($cmdname, $pname, $cvalue) = @_;
6692
6693     my $cfg = PVE::Storage::config();
6694
6695     my $storeid;
6696
6697     if ($cvalue =~ m/^([^:]+):/) {
6698         $storeid = $1;
6699     }
6700
6701     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6702
6703     my $res = [];
6704     foreach my $id (keys %$data) {
6705         foreach my $item (@{$data->{$id}}) {
6706             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6707             push @$res, $item->{volid} if defined($item->{volid});
6708         }
6709     }
6710
6711     return $res;
6712 }
6713
6714 my $complete_vmid_full = sub {
6715     my ($running) = @_;
6716
6717     my $idlist = vmstatus();
6718
6719     my $res = [];
6720
6721     foreach my $id (keys %$idlist) {
6722         my $d = $idlist->{$id};
6723         if (defined($running)) {
6724             next if $d->{template};
6725             next if $running && $d->{status} ne 'running';
6726             next if !$running && $d->{status} eq 'running';
6727         }
6728         push @$res, $id;
6729
6730     }
6731     return $res;
6732 };
6733
6734 sub complete_vmid {
6735     return &$complete_vmid_full();
6736 }
6737
6738 sub complete_vmid_stopped {
6739     return &$complete_vmid_full(0);
6740 }
6741
6742 sub complete_vmid_running {
6743     return &$complete_vmid_full(1);
6744 }
6745
6746 sub complete_storage {
6747
6748     my $cfg = PVE::Storage::config();
6749     my $ids = $cfg->{ids};
6750
6751     my $res = [];
6752     foreach my $sid (keys %$ids) {
6753         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6754         next if !$ids->{$sid}->{content}->{images};
6755         push @$res, $sid;
6756     }
6757
6758     return $res;
6759 }
6760
6761 1;