PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540     searchdomain => {
 541         optional => 1,
 542         type => 'string',
 543         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 544     },
 545     nameserver => {
 546         optional => 1,
 547         type => 'string', format => 'address-list',
 548         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 549     },
 550     sshkeys => {
 551         optional => 1,
 552         type => 'string',
 553         format => 'urlencoded',
 554         description => "cloud-init : Setup public SSH keys (one key per line, " .
 555                         "OpenSSH format).",
 556     },
 557     hostname => {
 558         optional => 1,
 559         description => "cloud-init: Hostname to use instead of the vm-name + search-domain.",
 560         type => 'string', format => 'dns-name',
 561         maxLength => 255,
 562     },
 563 };
 564
 565 # what about other qemu settings ?
 566 #cpu => 'string',
 567 #machine => 'string',
 568 #fda => 'file',
 569 #fdb => 'file',
 570 #mtdblock => 'file',
 571 #sd => 'file',
 572 #pflash => 'file',
 573 #snapshot => 'bool',
 574 #bootp => 'file',
 575 ##tftp => 'dir',
 576 ##smb => 'dir',
 577 #kernel => 'file',
 578 #append => 'string',
 579 #initrd => 'file',
 580 ##soundhw => 'string',
 581
 582 while (my ($k, $v) = each %$confdesc) {
 583     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 584 }
 585
 586 my $MAX_IDE_DISKS = 4;
 587 my $MAX_SCSI_DISKS = 14;
 588 my $MAX_VIRTIO_DISKS = 16;
 589 my $MAX_SATA_DISKS = 6;
 590 my $MAX_USB_DEVICES = 5;
 591 my $MAX_NETS = 32;
 592 my $MAX_UNUSED_DISKS = 8;
 593 my $MAX_HOSTPCI_DEVICES = 4;
 594 my $MAX_SERIAL_PORTS = 4;
 595 my $MAX_PARALLEL_PORTS = 3;
 596 my $MAX_NUMA = 8;
 597
 598 my $numa_fmt = {
 599     cpus => {
 600         type => "string",
 601         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 602         description => "CPUs accessing this NUMA node.",
 603         format_description => "id[-id];...",
 604     },
 605     memory => {
 606         type => "number",
 607         description => "Amount of memory this NUMA node provides.",
 608         optional => 1,
 609     },
 610     hostnodes => {
 611         type => "string",
 612         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 613         description => "Host NUMA nodes to use.",
 614         format_description => "id[-id];...",
 615         optional => 1,
 616     },
 617     policy => {
 618         type => 'string',
 619         enum => [qw(preferred bind interleave)],
 620         description => "NUMA allocation policy.",
 621         optional => 1,
 622     },
 623 };
 624 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 625 my $numadesc = {
 626     optional => 1,
 627     type => 'string', format => $numa_fmt,
 628     description => "NUMA topology.",
 629 };
 630 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 631
 632 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 633     $confdesc->{"numa$i"} = $numadesc;
 634 }
 635
 636 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 637                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 638                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 639 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 640
 641 my $net_fmt_bridge_descr = <<__EOD__;
 642 Bridge to attach the network device to. The Proxmox VE standard bridge
 643 is called 'vmbr0'.
 644
 645 If you do not specify a bridge, we create a kvm user (NATed) network
 646 device, which provides DHCP and DNS services. The following addresses
 647 are used:
 648
 649  10.0.2.2   Gateway
 650  10.0.2.3   DNS Server
 651  10.0.2.4   SMB Server
 652
 653 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 654 __EOD__
 655
 656 my $net_fmt = {
 657     macaddr => {
 658         type => 'string',
 659         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 660         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 661         format_description => "XX:XX:XX:XX:XX:XX",
 662         optional => 1,
 663     },
 664     model => {
 665         type => 'string',
 666         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 667         enum => $nic_model_list,
 668         default_key => 1,
 669     },
 670     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 671     bridge => {
 672         type => 'string',
 673         description => $net_fmt_bridge_descr,
 674         format_description => 'bridge',
 675         optional => 1,
 676     },
 677     queues => {
 678         type => 'integer',
 679         minimum => 0, maximum => 16,
 680         description => 'Number of packet queues to be used on the device.',
 681         optional => 1,
 682     },
 683     rate => {
 684         type => 'number',
 685         minimum => 0,
 686         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 687         optional => 1,
 688     },
 689     tag => {
 690         type => 'integer',
 691         minimum => 1, maximum => 4094,
 692         description => 'VLAN tag to apply to packets on this interface.',
 693         optional => 1,
 694     },
 695     trunks => {
 696         type => 'string',
 697         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 698         description => 'VLAN trunks to pass through this interface.',
 699         format_description => 'vlanid[;vlanid...]',
 700         optional => 1,
 701     },
 702     firewall => {
 703         type => 'boolean',
 704         description => 'Whether this interface should be protected by the firewall.',
 705         optional => 1,
 706     },
 707     link_down => {
 708         type => 'boolean',
 709         description => 'Whether this interface should be disconnected (like pulling the plug).',
 710         optional => 1,
 711     },
 712 };
 713
 714 my $netdesc = {
 715     optional => 1,
 716     type => 'string', format => $net_fmt,
 717     description => "Specify network devices.",
 718 };
 719
 720 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 721
 722 my $ipconfig_fmt = {
 723     ip => {
 724         type => 'string',
 725         format => 'pve-ipv4-config',
 726         format_description => 'IPv4Format/CIDR',
 727         description => 'IPv4 address in CIDR format.',
 728         optional => 1,
 729         default => 'dhcp',
 730     },
 731     gw => {
 732         type => 'string',
 733         format => 'ipv4',
 734         format_description => 'GatewayIPv4',
 735         description => 'Default gateway for IPv4 traffic.',
 736         optional => 1,
 737         requires => 'ip',
 738     },
 739     ip6 => {
 740         type => 'string',
 741         format => 'pve-ipv6-config',
 742         format_description => 'IPv6Format/CIDR',
 743         description => 'IPv6 address in CIDR format.',
 744         optional => 1,
 745         default => 'dhcp',
 746     },
 747     gw6 => {
 748         type => 'string',
 749         format => 'ipv6',
 750         format_description => 'GatewayIPv6',
 751         description => 'Default gateway for IPv6 traffic.',
 752         optional => 1,
 753         requires => 'ip6',
 754     },
 755 };
 756 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 757 my $ipconfigdesc = {
 758     optional => 1,
 759     type => 'string', format => 'pve-qm-ipconfig',
 760     description => <<'EODESCR',
 761 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 762
 763 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 764
 765 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 766 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 767
 768 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 769 EODESCR
 770 };
 771 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 772
 773 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 774     $confdesc->{"net$i"} = $netdesc;
 775     $confdesc->{"ipconfig$i"} = $ipconfigdesc;
 776 }
 777
 778 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 779 sub verify_volume_id_or_qm_path {
 780     my ($volid, $noerr) = @_;
 781
 782     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 783         return $volid;
 784     }
 785
 786     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 787     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 788     if ($@) {
 789         return undef if $noerr;
 790         die $@;
 791     }
 792     return $volid;
 793 }
 794
 795 my $drivename_hash;
 796
 797 my %drivedesc_base = (
 798     volume => { alias => 'file' },
 799     file => {
 800         type => 'string',
 801         format => 'pve-volume-id-or-qm-path',
 802         default_key => 1,
 803         format_description => 'volume',
 804         description => "The drive's backing volume.",
 805     },
 806     media => {
 807         type => 'string',
 808         enum => [qw(cdrom disk)],
 809         description => "The drive's media type.",
 810         default => 'disk',
 811         optional => 1
 812     },
 813     cyls => {
 814         type => 'integer',
 815         description => "Force the drive's physical geometry to have a specific cylinder count.",
 816         optional => 1
 817     },
 818     heads => {
 819         type => 'integer',
 820         description => "Force the drive's physical geometry to have a specific head count.",
 821         optional => 1
 822     },
 823     secs => {
 824         type => 'integer',
 825         description => "Force the drive's physical geometry to have a specific sector count.",
 826         optional => 1
 827     },
 828     trans => {
 829         type => 'string',
 830         enum => [qw(none lba auto)],
 831         description => "Force disk geometry bios translation mode.",
 832         optional => 1,
 833     },
 834     snapshot => {
 835         type => 'boolean',
 836         description => "Controls qemu's snapshot mode feature."
 837             . " If activated, changes made to the disk are temporary and will"
 838             . " be discarded when the VM is shutdown.",
 839         optional => 1,
 840     },
 841     cache => {
 842         type => 'string',
 843         enum => [qw(none writethrough writeback unsafe directsync)],
 844         description => "The drive's cache mode",
 845         optional => 1,
 846     },
 847     format => get_standard_option('pve-qm-image-format'),
 848     size => {
 849         type => 'string',
 850         format => 'disk-size',
 851         format_description => 'DiskSize',
 852         description => "Disk size. This is purely informational and has no effect.",
 853         optional => 1,
 854     },
 855     backup => {
 856         type => 'boolean',
 857         description => "Whether the drive should be included when making backups.",
 858         optional => 1,
 859     },
 860     replicate => {
 861         type => 'boolean',
 862         description => 'Whether the drive should considered for replication jobs.',
 863         optional => 1,
 864         default => 1,
 865     },
 866     rerror => {
 867         type => 'string',
 868         enum => [qw(ignore report stop)],
 869         description => 'Read error action.',
 870         optional => 1,
 871     },
 872     werror => {
 873         type => 'string',
 874         enum => [qw(enospc ignore report stop)],
 875         description => 'Write error action.',
 876         optional => 1,
 877     },
 878     aio => {
 879         type => 'string',
 880         enum => [qw(native threads)],
 881         description => 'AIO type to use.',
 882         optional => 1,
 883     },
 884     discard => {
 885         type => 'string',
 886         enum => [qw(ignore on)],
 887         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 888         optional => 1,
 889     },
 890     detect_zeroes => {
 891         type => 'boolean',
 892         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 893         optional => 1,
 894     },
 895     serial => {
 896         type => 'string',
 897         format => 'urlencoded',
 898         format_description => 'serial',
 899         maxLength => 20*3, # *3 since it's %xx url enoded
 900         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 901         optional => 1,
 902     },
 903     shared => {
 904         type => 'boolean',
 905         description => 'Mark this locally-managed volume as available on all nodes',
 906         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 907         optional => 1,
 908         default => 0,
 909     }
 910 );
 911
 912 my %iothread_fmt = ( iothread => {
 913         type => 'boolean',
 914         description => "Whether to use iothreads for this drive",
 915         optional => 1,
 916 });
 917
 918 my %model_fmt = (
 919     model => {
 920         type => 'string',
 921         format => 'urlencoded',
 922         format_description => 'model',
 923         maxLength => 40*3, # *3 since it's %xx url enoded
 924         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 925         optional => 1,
 926     },
 927 );
 928
 929 my %queues_fmt = (
 930     queues => {
 931         type => 'integer',
 932         description => "Number of queues.",
 933         minimum => 2,
 934         optional => 1
 935     }
 936 );
 937
 938 my %scsiblock_fmt = (
 939     scsiblock => {
 940         type => 'boolean',
 941         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 942         optional => 1,
 943         default => 0,
 944     },
 945 );
 946
 947 my $add_throttle_desc = sub {
 948     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 949     my $d = {
 950         type => $type,
 951         format_description => $unit,
 952         description => "Maximum $what in $longunit.",
 953         optional => 1,
 954     };
 955     $d->{minimum} = $minimum if defined($minimum);
 956     $drivedesc_base{$key} = $d;
 957 };
 958 # throughput: (leaky bucket)
 959 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 960 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 961 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 962 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 963 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 964 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 965 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 966 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 967 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 968
 969 # pools: (pool of IO before throttling starts taking effect)
 970 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 971 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 972 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 973 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 974 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 975 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 976
 977 # burst lengths
 978 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 979 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 980 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 981 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 982 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 983 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 984
 985 # legacy support
 986 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 987 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 988 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 989 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 990
 991 my $ide_fmt = {
 992     %drivedesc_base,
 993     %model_fmt,
 994 };
 995 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 996
 997 my $idedesc = {
 998     optional => 1,
 999     type => 'string', format => $ide_fmt,
1000     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1001 };
1002 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1003
1004 my $scsi_fmt = {
1005     %drivedesc_base,
1006     %iothread_fmt,
1007     %queues_fmt,
1008     %scsiblock_fmt,
1009 };
1010 my $scsidesc = {
1011     optional => 1,
1012     type => 'string', format => $scsi_fmt,
1013     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1014 };
1015 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1016
1017 my $sata_fmt = {
1018     %drivedesc_base,
1019 };
1020 my $satadesc = {
1021     optional => 1,
1022     type => 'string', format => $sata_fmt,
1023     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1024 };
1025 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1026
1027 my $virtio_fmt = {
1028     %drivedesc_base,
1029     %iothread_fmt,
1030 };
1031 my $virtiodesc = {
1032     optional => 1,
1033     type => 'string', format => $virtio_fmt,
1034     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1037
1038 my $alldrive_fmt = {
1039     %drivedesc_base,
1040     %iothread_fmt,
1041     %model_fmt,
1042     %queues_fmt,
1043     %scsiblock_fmt,
1044 };
1045
1046 my $efidisk_fmt = {
1047     volume => { alias => 'file' },
1048     file => {
1049         type => 'string',
1050         format => 'pve-volume-id-or-qm-path',
1051         default_key => 1,
1052         format_description => 'volume',
1053         description => "The drive's backing volume.",
1054     },
1055     format => get_standard_option('pve-qm-image-format'),
1056     size => {
1057         type => 'string',
1058         format => 'disk-size',
1059         format_description => 'DiskSize',
1060         description => "Disk size. This is purely informational and has no effect.",
1061         optional => 1,
1062     },
1063 };
1064
1065 my $efidisk_desc = {
1066     optional => 1,
1067     type => 'string', format => $efidisk_fmt,
1068     description => "Configure a Disk for storing EFI vars",
1069 };
1070
1071 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1072
1073 my $usb_fmt = {
1074     host => {
1075         default_key => 1,
1076         type => 'string', format => 'pve-qm-usb-device',
1077         format_description => 'HOSTUSBDEVICE|spice',
1078         description => <<EODESCR,
1079 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1080
1081  'bus-port(.port)*' (decimal numbers) or
1082  'vendor_id:product_id' (hexadeciaml numbers) or
1083  'spice'
1084
1085 You can use the 'lsusb -t' command to list existing usb devices.
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1088
1089 The value 'spice' can be used to add a usb redirection devices for spice.
1090 EODESCR
1091     },
1092     usb3 => {
1093         optional => 1,
1094         type => 'boolean',
1095         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1096         default => 0,
1097     },
1098 };
1099
1100 my $usbdesc = {
1101     optional => 1,
1102     type => 'string', format => $usb_fmt,
1103     description => "Configure an USB device (n is 0 to 4).",
1104 };
1105 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1106
1107 # NOTE: the match-groups of this regex are used in parse_hostpci
1108 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1109 my $hostpci_fmt = {
1110     host => {
1111         default_key => 1,
1112         type => 'string',
1113         pattern => qr/$PCIRE(;$PCIRE)*/,
1114         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1115         description => <<EODESCR,
1116 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1117 of PCI virtual functions of the host. HOSTPCIID syntax is:
1118
1119 'bus:dev.func' (hexadecimal numbers)
1120
1121 You can us the 'lspci' command to list existing PCI devices.
1122 EODESCR
1123     },
1124     rombar => {
1125         type => 'boolean',
1126         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1127         optional => 1,
1128         default => 1,
1129     },
1130     romfile => {
1131         type => 'string',
1132         pattern => '[^,;]+',
1133         format_description => 'string',
1134         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1135         optional => 1,
1136     },
1137     pcie => {
1138         type => 'boolean',
1139         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1140         optional => 1,
1141         default => 0,
1142     },
1143     'x-vga' => {
1144         type => 'boolean',
1145         description =>  "Enable vfio-vga device support.",
1146         optional => 1,
1147         default => 0,
1148     },
1149 };
1150 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1151
1152 my $hostpcidesc = {
1153         optional => 1,
1154         type => 'string', format => 'pve-qm-hostpci',
1155         description => "Map host PCI devices into guest.",
1156         verbose_description =>  <<EODESCR,
1157 Map host PCI devices into guest.
1158
1159 NOTE: This option allows direct access to host hardware. So it is no longer
1160 possible to migrate such machines - use with special care.
1161
1162 CAUTION: Experimental! User reported problems with this option.
1163 EODESCR
1164 };
1165 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1166
1167 my $serialdesc = {
1168         optional => 1,
1169         type => 'string',
1170         pattern => '(/dev/.+|socket)',
1171         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1172         verbose_description =>  <<EODESCR,
1173 Create a serial device inside the VM (n is 0 to 3), and pass through a
1174 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1175 host side (use 'qm terminal' to open a terminal connection).
1176
1177 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1178
1179 CAUTION: Experimental! User reported problems with this option.
1180 EODESCR
1181 };
1182
1183 my $paralleldesc= {
1184         optional => 1,
1185         type => 'string',
1186         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1187         description =>  "Map host parallel devices (n is 0 to 2).",
1188         verbose_description =>  <<EODESCR,
1189 Map host parallel devices (n is 0 to 2).
1190
1191 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1192
1193 CAUTION: Experimental! User reported problems with this option.
1194 EODESCR
1195 };
1196
1197 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1198     $confdesc->{"parallel$i"} = $paralleldesc;
1199 }
1200
1201 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1202     $confdesc->{"serial$i"} = $serialdesc;
1203 }
1204
1205 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1206     $confdesc->{"hostpci$i"} = $hostpcidesc;
1207 }
1208
1209 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1210     $drivename_hash->{"ide$i"} = 1;
1211     $confdesc->{"ide$i"} = $idedesc;
1212 }
1213
1214 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1215     $drivename_hash->{"sata$i"} = 1;
1216     $confdesc->{"sata$i"} = $satadesc;
1217 }
1218
1219 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1220     $drivename_hash->{"scsi$i"} = 1;
1221     $confdesc->{"scsi$i"} = $scsidesc ;
1222 }
1223
1224 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1225     $drivename_hash->{"virtio$i"} = 1;
1226     $confdesc->{"virtio$i"} = $virtiodesc;
1227 }
1228
1229 $drivename_hash->{efidisk0} = 1;
1230 $confdesc->{efidisk0} = $efidisk_desc;
1231
1232 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1233     $confdesc->{"usb$i"} = $usbdesc;
1234 }
1235
1236 my $unuseddesc = {
1237     optional => 1,
1238     type => 'string', format => 'pve-volume-id',
1239     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1240 };
1241
1242 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1243     $confdesc->{"unused$i"} = $unuseddesc;
1244 }
1245
1246 my $kvm_api_version = 0;
1247
1248 sub kvm_version {
1249
1250     return $kvm_api_version if $kvm_api_version;
1251
1252     my $fh = IO::File->new("</dev/kvm") ||
1253         return 0;
1254
1255     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1256         $kvm_api_version = $v;
1257     }
1258
1259     $fh->close();
1260
1261     return  $kvm_api_version;
1262 }
1263
1264 my $kvm_user_version;
1265
1266 sub kvm_user_version {
1267
1268     return $kvm_user_version if $kvm_user_version;
1269
1270     $kvm_user_version = 'unknown';
1271
1272     my $code = sub {
1273         my $line = shift;
1274         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1275             $kvm_user_version = $2;
1276         }
1277     };
1278
1279     eval { run_command("kvm -version", outfunc => $code); };
1280     warn $@ if $@;
1281
1282     return $kvm_user_version;
1283
1284 }
1285
1286 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1287
1288 sub valid_drive_names {
1289     # order is important - used to autoselect boot disk
1290     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1291             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1292             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1293             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1294             'efidisk0');
1295 }
1296
1297 sub is_valid_drivename {
1298     my $dev = shift;
1299
1300     return defined($drivename_hash->{$dev});
1301 }
1302
1303 sub option_exists {
1304     my $key = shift;
1305     return defined($confdesc->{$key});
1306 }
1307
1308 sub nic_models {
1309     return $nic_model_list;
1310 }
1311
1312 sub os_list_description {
1313
1314     return {
1315         other => 'Other',
1316         wxp => 'Windows XP',
1317         w2k => 'Windows 2000',
1318         w2k3 =>, 'Windows 2003',
1319         w2k8 => 'Windows 2008',
1320         wvista => 'Windows Vista',
1321         win7 => 'Windows 7',
1322         win8 => 'Windows 8/2012',
1323         win10 => 'Windows 10/2016',
1324         l24 => 'Linux 2.4',
1325         l26 => 'Linux 2.6',
1326     };
1327 }
1328
1329 my $cdrom_path;
1330
1331 sub get_cdrom_path {
1332
1333     return  $cdrom_path if $cdrom_path;
1334
1335     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1336     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1337     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1338 }
1339
1340 sub get_iso_path {
1341     my ($storecfg, $vmid, $cdrom) = @_;
1342
1343     if ($cdrom eq 'cdrom') {
1344         return get_cdrom_path();
1345     } elsif ($cdrom eq 'none') {
1346         return '';
1347     } elsif ($cdrom =~ m|^/|) {
1348         return $cdrom;
1349     } else {
1350         return PVE::Storage::path($storecfg, $cdrom);
1351     }
1352 }
1353
1354 # try to convert old style file names to volume IDs
1355 sub filename_to_volume_id {
1356     my ($vmid, $file, $media) = @_;
1357
1358      if (!($file eq 'none' || $file eq 'cdrom' ||
1359           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1360
1361         return undef if $file =~ m|/|;
1362
1363         if ($media && $media eq 'cdrom') {
1364             $file = "local:iso/$file";
1365         } else {
1366             $file = "local:$vmid/$file";
1367         }
1368     }
1369
1370     return $file;
1371 }
1372
1373 sub verify_media_type {
1374     my ($opt, $vtype, $media) = @_;
1375
1376     return if !$media;
1377
1378     my $etype;
1379     if ($media eq 'disk') {
1380         $etype = 'images';
1381     } elsif ($media eq 'cdrom') {
1382         $etype = 'iso';
1383     } else {
1384         die "internal error";
1385     }
1386
1387     return if ($vtype eq $etype);
1388
1389     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1390 }
1391
1392 sub cleanup_drive_path {
1393     my ($opt, $storecfg, $drive) = @_;
1394
1395     # try to convert filesystem paths to volume IDs
1396
1397     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1398         ($drive->{file} !~ m|^/dev/.+|) &&
1399         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1400         ($drive->{file} !~ m/^\d+$/)) {
1401         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1402         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1403         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1404         verify_media_type($opt, $vtype, $drive->{media});
1405         $drive->{file} = $volid;
1406     }
1407
1408     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1409 }
1410
1411 sub parse_hotplug_features {
1412     my ($data) = @_;
1413
1414     my $res = {};
1415
1416     return $res if $data eq '0';
1417
1418     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1419
1420     foreach my $feature (PVE::Tools::split_list($data)) {
1421         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1422             $res->{$1} = 1;
1423         } else {
1424             die "invalid hotplug feature '$feature'\n";
1425         }
1426     }
1427     return $res;
1428 }
1429
1430 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1431 sub pve_verify_hotplug_features {
1432     my ($value, $noerr) = @_;
1433
1434     return $value if parse_hotplug_features($value);
1435
1436     return undef if $noerr;
1437
1438     die "unable to parse hotplug option\n";
1439 }
1440
1441 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1442 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1443 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1444 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1445 #        [,iothread=on][,serial=serial][,model=model]
1446
1447 sub parse_drive {
1448     my ($key, $data) = @_;
1449
1450     my ($interface, $index);
1451
1452     if ($key =~ m/^([^\d]+)(\d+)$/) {
1453         $interface = $1;
1454         $index = $2;
1455     } else {
1456         return undef;
1457     }
1458
1459     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1460                                      : $confdesc->{$key}->{format};
1461     if (!$desc) {
1462         warn "invalid drive key: $key\n";
1463         return undef;
1464     }
1465     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1466     return undef if !$res;
1467     $res->{interface} = $interface;
1468     $res->{index} = $index;
1469
1470     my $error = 0;
1471     foreach my $opt (qw(bps bps_rd bps_wr)) {
1472         if (my $bps = defined(delete $res->{$opt})) {
1473             if (defined($res->{"m$opt"})) {
1474                 warn "both $opt and m$opt specified\n";
1475                 ++$error;
1476                 next;
1477             }
1478             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1479         }
1480     }
1481
1482     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1483     for my $requirement (
1484         [mbps_max => 'mbps'],
1485         [mbps_rd_max => 'mbps_rd'],
1486         [mbps_wr_max => 'mbps_wr'],
1487         [miops_max => 'miops'],
1488         [miops_rd_max => 'miops_rd'],
1489         [miops_wr_max => 'miops_wr'],
1490         [bps_max_length => 'mbps_max'],
1491         [bps_rd_max_length => 'mbps_rd_max'],
1492         [bps_wr_max_length => 'mbps_wr_max'],
1493         [iops_max_length => 'iops_max'],
1494         [iops_rd_max_length => 'iops_rd_max'],
1495         [iops_wr_max_length => 'iops_wr_max']) {
1496         my ($option, $requires) = @$requirement;
1497         if ($res->{$option} && !$res->{$requires}) {
1498             warn "$option requires $requires\n";
1499             ++$error;
1500         }
1501     }
1502
1503     return undef if $error;
1504
1505     return undef if $res->{mbps_rd} && $res->{mbps};
1506     return undef if $res->{mbps_wr} && $res->{mbps};
1507     return undef if $res->{iops_rd} && $res->{iops};
1508     return undef if $res->{iops_wr} && $res->{iops};
1509
1510     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1511         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1512         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1513         return undef if $res->{interface} eq 'virtio';
1514     }
1515
1516     if (my $size = $res->{size}) {
1517         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1518     }
1519
1520     return $res;
1521 }
1522
1523 sub print_drive {
1524     my ($vmid, $drive) = @_;
1525     my $data = { %$drive };
1526     delete $data->{$_} for qw(index interface);
1527     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1528 }
1529
1530 sub scsi_inquiry {
1531     my($fh, $noerr) = @_;
1532
1533     my $SG_IO = 0x2285;
1534     my $SG_GET_VERSION_NUM = 0x2282;
1535
1536     my $versionbuf = "\x00" x 8;
1537     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1538     if (!$ret) {
1539         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1540         return undef;
1541     }
1542     my $version = unpack("I", $versionbuf);
1543     if ($version < 30000) {
1544         die "scsi generic interface too old\n"  if !$noerr;
1545         return undef;
1546     }
1547
1548     my $buf = "\x00" x 36;
1549     my $sensebuf = "\x00" x 8;
1550     my $cmd = pack("C x3 C x1", 0x12, 36);
1551
1552     # see /usr/include/scsi/sg.h
1553     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1554
1555     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1556                       length($sensebuf), 0, length($buf), $buf,
1557                       $cmd, $sensebuf, 6000);
1558
1559     $ret = ioctl($fh, $SG_IO, $packet);
1560     if (!$ret) {
1561         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1562         return undef;
1563     }
1564
1565     my @res = unpack($sg_io_hdr_t, $packet);
1566     if ($res[17] || $res[18]) {
1567         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1568         return undef;
1569     }
1570
1571     my $res = {};
1572     (my $byte0, my $byte1, $res->{vendor},
1573      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1574
1575     $res->{removable} = $byte1 & 128 ? 1 : 0;
1576     $res->{type} = $byte0 & 31;
1577
1578     return $res;
1579 }
1580
1581 sub path_is_scsi {
1582     my ($path) = @_;
1583
1584     my $fh = IO::File->new("+<$path") || return undef;
1585     my $res = scsi_inquiry($fh, 1);
1586     close($fh);
1587
1588     return $res;
1589 }
1590
1591 sub machine_type_is_q35 {
1592     my ($conf) = @_;
1593
1594     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1595 }
1596
1597 sub print_tabletdevice_full {
1598     my ($conf) = @_;
1599
1600     my $q35 = machine_type_is_q35($conf);
1601
1602     # we use uhci for old VMs because tablet driver was buggy in older qemu
1603     my $usbbus = $q35 ? "ehci" : "uhci";
1604
1605     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1606 }
1607
1608 sub print_drivedevice_full {
1609     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1610
1611     my $device = '';
1612     my $maxdev = 0;
1613
1614     if ($drive->{interface} eq 'virtio') {
1615         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1616         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1617         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1618     } elsif ($drive->{interface} eq 'scsi') {
1619
1620         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1621         my $unit = $drive->{index} % $maxdev;
1622         my $devicetype = 'hd';
1623         my $path = '';
1624         if (drive_is_cdrom($drive)) {
1625             $devicetype = 'cd';
1626         } else {
1627             if ($drive->{file} =~ m|^/|) {
1628                 $path = $drive->{file};
1629                 if (my $info = path_is_scsi($path)) {
1630                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1631                         $devicetype = 'block';
1632                     } elsif ($info->{type} == 1) { # tape
1633                         $devicetype = 'generic';
1634                     }
1635                 }
1636             } else {
1637                  $path = PVE::Storage::path($storecfg, $drive->{file});
1638             }
1639
1640             if($path =~ m/^iscsi\:\/\//){
1641                 $devicetype = 'generic';
1642             }
1643         }
1644
1645         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1646             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1647         } else {
1648             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1649         }
1650
1651     } elsif ($drive->{interface} eq 'ide'){
1652         $maxdev = 2;
1653         my $controller = int($drive->{index} / $maxdev);
1654         my $unit = $drive->{index} % $maxdev;
1655         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1656
1657         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1658         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1659             $model = URI::Escape::uri_unescape($model);
1660             $device .= ",model=$model";
1661         }
1662     } elsif ($drive->{interface} eq 'sata'){
1663         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1664         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1665         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1666     } elsif ($drive->{interface} eq 'usb') {
1667         die "implement me";
1668         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1669     } else {
1670         die "unsupported interface type";
1671     }
1672
1673     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1674
1675     return $device;
1676 }
1677
1678 sub get_initiator_name {
1679     my $initiator;
1680
1681     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1682     while (defined(my $line = <$fh>)) {
1683         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1684         $initiator = $1;
1685         last;
1686     }
1687     $fh->close();
1688
1689     return $initiator;
1690 }
1691
1692 sub print_drive_full {
1693     my ($storecfg, $vmid, $drive) = @_;
1694
1695     my $path;
1696     my $volid = $drive->{file};
1697     my $format;
1698
1699     if (drive_is_cdrom($drive)) {
1700         $path = get_iso_path($storecfg, $vmid, $volid);
1701     } else {
1702         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1703         if ($storeid) {
1704             $path = PVE::Storage::path($storecfg, $volid);
1705             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1706             $format = qemu_img_format($scfg, $volname);
1707         } else {
1708             $path = $volid;
1709             $format = "raw";
1710         }
1711    }
1712
1713     my $opts = '';
1714     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1715     foreach my $o (@qemu_drive_options) {
1716         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1717     }
1718
1719     # snapshot only accepts on|off
1720     if (defined($drive->{snapshot})) {
1721         my $v = $drive->{snapshot} ? 'on' : 'off';
1722         $opts .= ",snapshot=$v";
1723     }
1724
1725     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1726         my ($dir, $qmpname) = @$type;
1727         if (my $v = $drive->{"mbps$dir"}) {
1728             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1729         }
1730         if (my $v = $drive->{"mbps${dir}_max"}) {
1731             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1732         }
1733         if (my $v = $drive->{"bps${dir}_max_length"}) {
1734             $opts .= ",throttling.bps$qmpname-max-length=$v";
1735         }
1736         if (my $v = $drive->{"iops${dir}"}) {
1737             $opts .= ",throttling.iops$qmpname=$v";
1738         }
1739         if (my $v = $drive->{"iops${dir}_max"}) {
1740             $opts .= ",throttling.iops$qmpname-max=$v";
1741         }
1742         if (my $v = $drive->{"iops${dir}_max_length"}) {
1743             $opts .= ",throttling.iops$qmpname-max-length=$v";
1744         }
1745     }
1746
1747     if (my $serial = $drive->{serial}) {
1748         $serial = URI::Escape::uri_unescape($serial);
1749         $opts .= ",serial=$serial";
1750     }
1751
1752     $opts .= ",format=$format" if $format && !$drive->{format};
1753
1754     my $cache_direct = 0;
1755
1756     if (my $cache = $drive->{cache}) {
1757         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1758     } elsif (!drive_is_cdrom($drive)) {
1759         $opts .= ",cache=none";
1760         $cache_direct = 1;
1761     }
1762
1763     # aio native works only with O_DIRECT
1764     if (!$drive->{aio}) {
1765         if($cache_direct) {
1766             $opts .= ",aio=native";
1767         } else {
1768             $opts .= ",aio=threads";
1769         }
1770     }
1771
1772     if (!drive_is_cdrom($drive)) {
1773         my $detectzeroes;
1774         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1775             $detectzeroes = 'off';
1776         } elsif ($drive->{discard}) {
1777             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1778         } else {
1779             # This used to be our default with discard not being specified:
1780             $detectzeroes = 'on';
1781         }
1782         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1783     }
1784
1785     my $pathinfo = $path ? "file=$path," : '';
1786
1787     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1788 }
1789
1790 sub print_netdevice_full {
1791     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1792
1793     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1794
1795     my $device = $net->{model};
1796     if ($net->{model} eq 'virtio') {
1797          $device = 'virtio-net-pci';
1798      };
1799
1800     my $pciaddr = print_pci_addr("$netid", $bridges);
1801     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1802     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1803         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1804         my $vectors = $net->{queues} * 2 + 2;
1805         $tmpstr .= ",vectors=$vectors,mq=on";
1806     }
1807     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1808
1809     if ($use_old_bios_files) {
1810         my $romfile;
1811         if ($device eq 'virtio-net-pci') {
1812             $romfile = 'pxe-virtio.rom';
1813         } elsif ($device eq 'e1000') {
1814             $romfile = 'pxe-e1000.rom';
1815         } elsif ($device eq 'ne2k') {
1816             $romfile = 'pxe-ne2k_pci.rom';
1817         } elsif ($device eq 'pcnet') {
1818             $romfile = 'pxe-pcnet.rom';
1819         } elsif ($device eq 'rtl8139') {
1820             $romfile = 'pxe-rtl8139.rom';
1821         }
1822         $tmpstr .= ",romfile=$romfile" if $romfile;
1823     }
1824
1825     return $tmpstr;
1826 }
1827
1828 sub print_netdev_full {
1829     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1830
1831     my $i = '';
1832     if ($netid =~ m/^net(\d+)$/) {
1833         $i = int($1);
1834     }
1835
1836     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1837
1838     my $ifname = "tap${vmid}i$i";
1839
1840     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1841     die "interface name '$ifname' is too long (max 15 character)\n"
1842         if length($ifname) >= 16;
1843
1844     my $vhostparam = '';
1845     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1846
1847     my $vmname = $conf->{name} || "vm$vmid";
1848
1849     my $netdev = "";
1850     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1851
1852     if ($net->{bridge}) {
1853         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1854     } else {
1855         $netdev = "type=user,id=$netid,hostname=$vmname";
1856     }
1857
1858     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1859
1860     return $netdev;
1861 }
1862
1863
1864 sub print_cpu_device {
1865     my ($conf, $id) = @_;
1866
1867     my $kvm = $conf->{kvm} // 1;
1868     my $cpu = $kvm ? "kvm64" : "qemu64";
1869     if (my $cputype = $conf->{cpu}) {
1870         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1871             or die "Cannot parse cpu description: $cputype\n";
1872         $cpu = $cpuconf->{cputype};
1873     }
1874
1875     my $cores = $conf->{cores} || 1;
1876
1877     my $current_core = ($id - 1) % $cores;
1878     my $current_socket = int(($id - 1 - $current_core)/$cores);
1879
1880     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1881 }
1882
1883 sub drive_is_cdrom {
1884     my ($drive, $exclude_cloudinit) = @_;
1885
1886     return 0 if $exclude_cloudinit && $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1887
1888     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1889
1890 }
1891
1892 sub parse_number_sets {
1893     my ($set) = @_;
1894     my $res = [];
1895     foreach my $part (split(/;/, $set)) {
1896         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1897             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1898             push @$res, [ $1, $2 ];
1899         } else {
1900             die "invalid range: $part\n";
1901         }
1902     }
1903     return $res;
1904 }
1905
1906 sub parse_numa {
1907     my ($data) = @_;
1908
1909     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1910     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1911     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1912     return $res;
1913 }
1914
1915 sub parse_hostpci {
1916     my ($value) = @_;
1917
1918     return undef if !$value;
1919
1920     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1921
1922     my @idlist = split(/;/, $res->{host});
1923     delete $res->{host};
1924     foreach my $id (@idlist) {
1925         if ($id =~ /^$PCIRE$/) {
1926             if (defined($2)) {
1927                 push @{$res->{pciid}}, { id => $1, function => $2 };
1928             } else {
1929                 my $pcidevices = lspci($1);
1930                 $res->{pciid} = $pcidevices->{$1};
1931             }
1932         } else {
1933             # should have been caught by parse_property_string already
1934             die "failed to parse PCI id: $id\n";
1935         }
1936     }
1937     return $res;
1938 }
1939
1940 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1941 sub parse_net {
1942     my ($data) = @_;
1943
1944     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1945     if ($@) {
1946         warn $@;
1947         return undef;
1948     }
1949     if (!defined($res->{macaddr})) {
1950         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1951         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1952     }
1953     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1954     return $res;
1955 }
1956
1957 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1958 sub parse_ipconfig {
1959     my ($data) = @_;
1960
1961     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1962     if ($@) {
1963         warn $@;
1964         return undef;
1965     }
1966
1967     if ($res->{gw} && !$res->{ip}) {
1968         warn 'gateway specified without specifying an IP address';
1969         return undef;
1970     }
1971     if ($res->{gw6} && !$res->{ip6}) {
1972         warn 'IPv6 gateway specified without specifying an IPv6 address';
1973         return undef;
1974     }
1975     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1976         warn 'gateway specified together with DHCP';
1977         return undef;
1978     }
1979     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1980         # gw6 + auto/dhcp
1981         warn "IPv6 gateway specified together with $res->{ip6} address";
1982         return undef;
1983     }
1984
1985     if (!$res->{ip} && !$res->{ip6}) {
1986         return { ip => 'dhcp', ip6 => 'dhcp' };
1987     }
1988
1989     return $res;
1990 }
1991
1992 sub print_net {
1993     my $net = shift;
1994
1995     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1996 }
1997
1998 sub add_random_macs {
1999     my ($settings) = @_;
2000
2001     foreach my $opt (keys %$settings) {
2002         next if $opt !~ m/^net(\d+)$/;
2003         my $net = parse_net($settings->{$opt});
2004         next if !$net;
2005         $settings->{$opt} = print_net($net);
2006     }
2007 }
2008
2009 sub vm_is_volid_owner {
2010     my ($storecfg, $vmid, $volid) = @_;
2011
2012     if ($volid !~  m|^/|) {
2013         my ($path, $owner);
2014         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2015         if ($owner && ($owner == $vmid)) {
2016             return 1;
2017         }
2018     }
2019
2020     return undef;
2021 }
2022
2023 sub split_flagged_list {
2024     my $text = shift || '';
2025     $text =~ s/[,;]/ /g;
2026     $text =~ s/^\s+//;
2027     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2028 }
2029
2030 sub join_flagged_list {
2031     my ($how, $lst) = @_;
2032     join $how, map { $lst->{$_} . $_ } keys %$lst;
2033 }
2034
2035 sub vmconfig_delete_pending_option {
2036     my ($conf, $key, $force) = @_;
2037
2038     delete $conf->{pending}->{$key};
2039     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2040     $pending_delete_hash->{$key} = $force ? '!' : '';
2041     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2042 }
2043
2044 sub vmconfig_undelete_pending_option {
2045     my ($conf, $key) = @_;
2046
2047     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2048     delete $pending_delete_hash->{$key};
2049
2050     if (%$pending_delete_hash) {
2051         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2052     } else {
2053         delete $conf->{pending}->{delete};
2054     }
2055 }
2056
2057 sub vmconfig_register_unused_drive {
2058     my ($storecfg, $vmid, $conf, $drive) = @_;
2059
2060     if (!drive_is_cdrom($drive, 1)) {
2061         my $volid = $drive->{file};
2062         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2063             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2064         }
2065     }
2066 }
2067
2068 sub vmconfig_cleanup_pending {
2069     my ($conf) = @_;
2070
2071     # remove pending changes when nothing changed
2072     my $changes;
2073     foreach my $opt (keys %{$conf->{pending}}) {
2074         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2075             $changes = 1;
2076             delete $conf->{pending}->{$opt};
2077         }
2078     }
2079
2080     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2081     my $pending_delete_hash = {};
2082     while (my ($opt, $force) = each %$current_delete_hash) {
2083         if (defined($conf->{$opt})) {
2084             $pending_delete_hash->{$opt} = $force;
2085         } else {
2086             $changes = 1;
2087         }
2088     }
2089
2090     if (%$pending_delete_hash) {
2091         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2092     } else {
2093         delete $conf->{pending}->{delete};
2094     }
2095
2096     return $changes;
2097 }
2098
2099 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2100 my $smbios1_fmt = {
2101     uuid => {
2102         type => 'string',
2103         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2104         format_description => 'UUID',
2105         description => "Set SMBIOS1 UUID.",
2106         optional => 1,
2107     },
2108     version => {
2109         type => 'string',
2110         pattern => '\S+',
2111         format_description => 'string',
2112         description => "Set SMBIOS1 version.",
2113         optional => 1,
2114     },
2115     serial => {
2116         type => 'string',
2117         pattern => '\S+',
2118         format_description => 'string',
2119         description => "Set SMBIOS1 serial number.",
2120         optional => 1,
2121     },
2122     manufacturer => {
2123         type => 'string',
2124         pattern => '\S+',
2125         format_description => 'string',
2126         description => "Set SMBIOS1 manufacturer.",
2127         optional => 1,
2128     },
2129     product => {
2130         type => 'string',
2131         pattern => '\S+',
2132         format_description => 'string',
2133         description => "Set SMBIOS1 product ID.",
2134         optional => 1,
2135     },
2136     sku => {
2137         type => 'string',
2138         pattern => '\S+',
2139         format_description => 'string',
2140         description => "Set SMBIOS1 SKU string.",
2141         optional => 1,
2142     },
2143     family => {
2144         type => 'string',
2145         pattern => '\S+',
2146         format_description => 'string',
2147         description => "Set SMBIOS1 family string.",
2148         optional => 1,
2149     },
2150 };
2151
2152 sub parse_smbios1 {
2153     my ($data) = @_;
2154
2155     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2156     warn $@ if $@;
2157     return $res;
2158 }
2159
2160 sub print_smbios1 {
2161     my ($smbios1) = @_;
2162     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2163 }
2164
2165 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2166
2167 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2168 sub verify_bootdisk {
2169     my ($value, $noerr) = @_;
2170
2171     return $value if is_valid_drivename($value);
2172
2173     return undef if $noerr;
2174
2175     die "invalid boot disk '$value'\n";
2176 }
2177
2178 sub parse_watchdog {
2179     my ($value) = @_;
2180
2181     return undef if !$value;
2182
2183     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2184     warn $@ if $@;
2185     return $res;
2186 }
2187
2188 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2189 sub verify_usb_device {
2190     my ($value, $noerr) = @_;
2191
2192     return $value if parse_usb_device($value);
2193
2194     return undef if $noerr;
2195
2196     die "unable to parse usb device\n";
2197 }
2198
2199 # add JSON properties for create and set function
2200 sub json_config_properties {
2201     my $prop = shift;
2202
2203     foreach my $opt (keys %$confdesc) {
2204         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2205         $prop->{$opt} = $confdesc->{$opt};
2206     }
2207
2208     return $prop;
2209 }
2210
2211 sub check_type {
2212     my ($key, $value) = @_;
2213
2214     die "unknown setting '$key'\n" if !$confdesc->{$key};
2215
2216     my $type = $confdesc->{$key}->{type};
2217
2218     if (!defined($value)) {
2219         die "got undefined value\n";
2220     }
2221
2222     if ($value =~ m/[\n\r]/) {
2223         die "property contains a line feed\n";
2224     }
2225
2226     if ($type eq 'boolean') {
2227         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2228         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2229         die "type check ('boolean') failed - got '$value'\n";
2230     } elsif ($type eq 'integer') {
2231         return int($1) if $value =~ m/^(\d+)$/;
2232         die "type check ('integer') failed - got '$value'\n";
2233     } elsif ($type eq 'number') {
2234         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2235         die "type check ('number') failed - got '$value'\n";
2236     } elsif ($type eq 'string') {
2237         if (my $fmt = $confdesc->{$key}->{format}) {
2238             PVE::JSONSchema::check_format($fmt, $value);
2239             return $value;
2240         }
2241         $value =~ s/^\"(.*)\"$/$1/;
2242         return $value;
2243     } else {
2244         die "internal error"
2245     }
2246 }
2247
2248 sub check_iommu_support{
2249     #fixme : need to check IOMMU support
2250     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2251
2252     my $iommu=1;
2253     return $iommu;
2254
2255 }
2256
2257 sub touch_config {
2258     my ($vmid) = @_;
2259
2260     my $conf = PVE::QemuConfig->config_file($vmid);
2261     utime undef, undef, $conf;
2262 }
2263
2264 sub destroy_vm {
2265     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2266
2267     my $conffile = PVE::QemuConfig->config_file($vmid);
2268
2269     my $conf = PVE::QemuConfig->load_config($vmid);
2270
2271     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2272
2273     if ($conf->{template}) {
2274         # check if any base image is still used by a linked clone
2275         foreach_drive($conf, sub {
2276                 my ($ds, $drive) = @_;
2277
2278                 return if drive_is_cdrom($drive);
2279
2280                 my $volid = $drive->{file};
2281
2282                 return if !$volid || $volid =~ m|^/|;
2283
2284                 die "base volume '$volid' is still in use by linked cloned\n"
2285                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2286
2287         });
2288     }
2289
2290     # only remove disks owned by this VM
2291     foreach_drive($conf, sub {
2292         my ($ds, $drive) = @_;
2293
2294         return if drive_is_cdrom($drive, 1);
2295
2296         my $volid = $drive->{file};
2297
2298         return if !$volid || $volid =~ m|^/|;
2299
2300         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2301         return if !$path || !$owner || ($owner != $vmid);
2302
2303         eval {
2304             PVE::Storage::vdisk_free($storecfg, $volid);
2305         };
2306         warn "Could not remove disk '$volid', check manually: $@" if $@;
2307
2308     });
2309
2310     if ($keep_empty_config) {
2311         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2312     } else {
2313         unlink $conffile;
2314     }
2315
2316     # also remove unused disk
2317     eval {
2318         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2319
2320         eval {
2321             PVE::Storage::foreach_volid($dl, sub {
2322                 my ($volid, $sid, $volname, $d) = @_;
2323                 PVE::Storage::vdisk_free($storecfg, $volid);
2324             });
2325         };
2326         warn $@ if $@;
2327
2328     };
2329     warn $@ if $@;
2330 }
2331
2332 sub parse_vm_config {
2333     my ($filename, $raw) = @_;
2334
2335     return undef if !defined($raw);
2336
2337     my $res = {
2338         digest => Digest::SHA::sha1_hex($raw),
2339         snapshots => {},
2340         pending => {},
2341     };
2342
2343     $filename =~ m|/qemu-server/(\d+)\.conf$|
2344         || die "got strange filename '$filename'";
2345
2346     my $vmid = $1;
2347
2348     my $conf = $res;
2349     my $descr;
2350     my $section = '';
2351
2352     my @lines = split(/\n/, $raw);
2353     foreach my $line (@lines) {
2354         next if $line =~ m/^\s*$/;
2355
2356         if ($line =~ m/^\[PENDING\]\s*$/i) {
2357             $section = 'pending';
2358             if (defined($descr)) {
2359                 $descr =~ s/\s+$//;
2360                 $conf->{description} = $descr;
2361             }
2362             $descr = undef;
2363             $conf = $res->{$section} = {};
2364             next;
2365
2366         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2367             $section = $1;
2368             if (defined($descr)) {
2369                 $descr =~ s/\s+$//;
2370                 $conf->{description} = $descr;
2371             }
2372             $descr = undef;
2373             $conf = $res->{snapshots}->{$section} = {};
2374             next;
2375         }
2376
2377         if ($line =~ m/^\#(.*)\s*$/) {
2378             $descr = '' if !defined($descr);
2379             $descr .= PVE::Tools::decode_text($1) . "\n";
2380             next;
2381         }
2382
2383         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2384             $descr = '' if !defined($descr);
2385             $descr .= PVE::Tools::decode_text($2);
2386         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2387             $conf->{snapstate} = $1;
2388         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2389             my $key = $1;
2390             my $value = $2;
2391             $conf->{$key} = $value;
2392         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2393             my $value = $1;
2394             if ($section eq 'pending') {
2395                 $conf->{delete} = $value; # we parse this later
2396             } else {
2397                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2398             }
2399         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2400             my $key = $1;
2401             my $value = $2;
2402             eval { $value = check_type($key, $value); };
2403             if ($@) {
2404                 warn "vm $vmid - unable to parse value of '$key' - $@";
2405             } else {
2406                 $key = 'ide2' if $key eq 'cdrom';
2407                 my $fmt = $confdesc->{$key}->{format};
2408                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2409                     my $v = parse_drive($key, $value);
2410                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2411                         $v->{file} = $volid;
2412                         $value = print_drive($vmid, $v);
2413                     } else {
2414                         warn "vm $vmid - unable to parse value of '$key'\n";
2415                         next;
2416                     }
2417                 }
2418
2419                 $conf->{$key} = $value;
2420             }
2421         }
2422     }
2423
2424     if (defined($descr)) {
2425         $descr =~ s/\s+$//;
2426         $conf->{description} = $descr;
2427     }
2428     delete $res->{snapstate}; # just to be sure
2429
2430     return $res;
2431 }
2432
2433 sub write_vm_config {
2434     my ($filename, $conf) = @_;
2435
2436     delete $conf->{snapstate}; # just to be sure
2437
2438     if ($conf->{cdrom}) {
2439         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2440         $conf->{ide2} = $conf->{cdrom};
2441         delete $conf->{cdrom};
2442     }
2443
2444     # we do not use 'smp' any longer
2445     if ($conf->{sockets}) {
2446         delete $conf->{smp};
2447     } elsif ($conf->{smp}) {
2448         $conf->{sockets} = $conf->{smp};
2449         delete $conf->{cores};
2450         delete $conf->{smp};
2451     }
2452
2453     my $used_volids = {};
2454
2455     my $cleanup_config = sub {
2456         my ($cref, $pending, $snapname) = @_;
2457
2458         foreach my $key (keys %$cref) {
2459             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2460                 $key eq 'snapstate' || $key eq 'pending';
2461             my $value = $cref->{$key};
2462             if ($key eq 'delete') {
2463                 die "propertry 'delete' is only allowed in [PENDING]\n"
2464                     if !$pending;
2465                 # fixme: check syntax?
2466                 next;
2467             }
2468             eval { $value = check_type($key, $value); };
2469             die "unable to parse value of '$key' - $@" if $@;
2470
2471             $cref->{$key} = $value;
2472
2473             if (!$snapname && is_valid_drivename($key)) {
2474                 my $drive = parse_drive($key, $value);
2475                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2476             }
2477         }
2478     };
2479
2480     &$cleanup_config($conf);
2481
2482     &$cleanup_config($conf->{pending}, 1);
2483
2484     foreach my $snapname (keys %{$conf->{snapshots}}) {
2485         die "internal error" if $snapname eq 'pending';
2486         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2487     }
2488
2489     # remove 'unusedX' settings if we re-add a volume
2490     foreach my $key (keys %$conf) {
2491         my $value = $conf->{$key};
2492         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2493             delete $conf->{$key};
2494         }
2495     }
2496
2497     my $generate_raw_config = sub {
2498         my ($conf, $pending) = @_;
2499
2500         my $raw = '';
2501
2502         # add description as comment to top of file
2503         if (defined(my $descr = $conf->{description})) {
2504             if ($descr) {
2505                 foreach my $cl (split(/\n/, $descr)) {
2506                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2507                 }
2508             } else {
2509                 $raw .= "#\n" if $pending;
2510             }
2511         }
2512
2513         foreach my $key (sort keys %$conf) {
2514             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2515             $raw .= "$key: $conf->{$key}\n";
2516         }
2517         return $raw;
2518     };
2519
2520     my $raw = &$generate_raw_config($conf);
2521
2522     if (scalar(keys %{$conf->{pending}})){
2523         $raw .= "\n[PENDING]\n";
2524         $raw .= &$generate_raw_config($conf->{pending}, 1);
2525     }
2526
2527     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2528         $raw .= "\n[$snapname]\n";
2529         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2530     }
2531
2532     return $raw;
2533 }
2534
2535 sub load_defaults {
2536
2537     my $res = {};
2538
2539     # we use static defaults from our JSON schema configuration
2540     foreach my $key (keys %$confdesc) {
2541         if (defined(my $default = $confdesc->{$key}->{default})) {
2542             $res->{$key} = $default;
2543         }
2544     }
2545
2546     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2547     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2548
2549     return $res;
2550 }
2551
2552 sub config_list {
2553     my $vmlist = PVE::Cluster::get_vmlist();
2554     my $res = {};
2555     return $res if !$vmlist || !$vmlist->{ids};
2556     my $ids = $vmlist->{ids};
2557
2558     foreach my $vmid (keys %$ids) {
2559         my $d = $ids->{$vmid};
2560         next if !$d->{node} || $d->{node} ne $nodename;
2561         next if !$d->{type} || $d->{type} ne 'qemu';
2562         $res->{$vmid}->{exists} = 1;
2563     }
2564     return $res;
2565 }
2566
2567 # test if VM uses local resources (to prevent migration)
2568 sub check_local_resources {
2569     my ($conf, $noerr) = @_;
2570
2571     my $loc_res = 0;
2572
2573     $loc_res = 1 if $conf->{hostusb}; # old syntax
2574     $loc_res = 1 if $conf->{hostpci}; # old syntax
2575
2576     foreach my $k (keys %$conf) {
2577         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2578         # sockets are safe: they will recreated be on the target side post-migrate
2579         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2580         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2581     }
2582
2583     die "VM uses local resources\n" if $loc_res && !$noerr;
2584
2585     return $loc_res;
2586 }
2587
2588 # check if used storages are available on all nodes (use by migrate)
2589 sub check_storage_availability {
2590     my ($storecfg, $conf, $node) = @_;
2591
2592     foreach_drive($conf, sub {
2593         my ($ds, $drive) = @_;
2594
2595         my $volid = $drive->{file};
2596         return if !$volid;
2597
2598         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2599         return if !$sid;
2600
2601         # check if storage is available on both nodes
2602         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2603         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2604    });
2605 }
2606
2607 # list nodes where all VM images are available (used by has_feature API)
2608 sub shared_nodes {
2609     my ($conf, $storecfg) = @_;
2610
2611     my $nodelist = PVE::Cluster::get_nodelist();
2612     my $nodehash = { map { $_ => 1 } @$nodelist };
2613     my $nodename = PVE::INotify::nodename();
2614
2615     foreach_drive($conf, sub {
2616         my ($ds, $drive) = @_;
2617
2618         my $volid = $drive->{file};
2619         return if !$volid;
2620
2621         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2622         if ($storeid) {
2623             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2624             if ($scfg->{disable}) {
2625                 $nodehash = {};
2626             } elsif (my $avail = $scfg->{nodes}) {
2627                 foreach my $node (keys %$nodehash) {
2628                     delete $nodehash->{$node} if !$avail->{$node};
2629                 }
2630             } elsif (!$scfg->{shared}) {
2631                 foreach my $node (keys %$nodehash) {
2632                     delete $nodehash->{$node} if $node ne $nodename
2633                 }
2634             }
2635         }
2636     });
2637
2638     return $nodehash
2639 }
2640
2641 sub check_cmdline {
2642     my ($pidfile, $pid) = @_;
2643
2644     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2645     if (defined($fh)) {
2646         my $line = <$fh>;
2647         $fh->close;
2648         return undef if !$line;
2649         my @param = split(/\0/, $line);
2650
2651         my $cmd = $param[0];
2652         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2653
2654         for (my $i = 0; $i < scalar (@param); $i++) {
2655             my $p = $param[$i];
2656             next if !$p;
2657             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2658                 my $p = $param[$i+1];
2659                 return 1 if $p && ($p eq $pidfile);
2660                 return undef;
2661             }
2662         }
2663     }
2664     return undef;
2665 }
2666
2667 sub check_running {
2668     my ($vmid, $nocheck, $node) = @_;
2669
2670     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2671
2672     die "unable to find configuration file for VM $vmid - no such machine\n"
2673         if !$nocheck && ! -f $filename;
2674
2675     my $pidfile = pidfile_name($vmid);
2676
2677     if (my $fd = IO::File->new("<$pidfile")) {
2678         my $st = stat($fd);
2679         my $line = <$fd>;
2680         close($fd);
2681
2682         my $mtime = $st->mtime;
2683         if ($mtime > time()) {
2684             warn "file '$filename' modified in future\n";
2685         }
2686
2687         if ($line =~ m/^(\d+)$/) {
2688             my $pid = $1;
2689             if (check_cmdline($pidfile, $pid)) {
2690                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2691                     return $pid;
2692                 }
2693             }
2694         }
2695     }
2696
2697     return undef;
2698 }
2699
2700 sub vzlist {
2701
2702     my $vzlist = config_list();
2703
2704     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2705
2706     while (defined(my $de = $fd->read)) {
2707         next if $de !~ m/^(\d+)\.pid$/;
2708         my $vmid = $1;
2709         next if !defined($vzlist->{$vmid});
2710         if (my $pid = check_running($vmid)) {
2711             $vzlist->{$vmid}->{pid} = $pid;
2712         }
2713     }
2714
2715     return $vzlist;
2716 }
2717
2718 sub disksize {
2719     my ($storecfg, $conf) = @_;
2720
2721     my $bootdisk = $conf->{bootdisk};
2722     return undef if !$bootdisk;
2723     return undef if !is_valid_drivename($bootdisk);
2724
2725     return undef if !$conf->{$bootdisk};
2726
2727     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2728     return undef if !defined($drive);
2729
2730     return undef if drive_is_cdrom($drive);
2731
2732     my $volid = $drive->{file};
2733     return undef if !$volid;
2734
2735     return $drive->{size};
2736 }
2737
2738 my $last_proc_pid_stat;
2739
2740 # get VM status information
2741 # This must be fast and should not block ($full == false)
2742 # We only query KVM using QMP if $full == true (this can be slow)
2743 sub vmstatus {
2744     my ($opt_vmid, $full) = @_;
2745
2746     my $res = {};
2747
2748     my $storecfg = PVE::Storage::config();
2749
2750     my $list = vzlist();
2751     my $defaults = load_defaults();
2752
2753     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2754
2755     my $cpucount = $cpuinfo->{cpus} || 1;
2756
2757     foreach my $vmid (keys %$list) {
2758         next if $opt_vmid && ($vmid ne $opt_vmid);
2759
2760         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2761         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2762
2763         my $d = {};
2764         $d->{pid} = $list->{$vmid}->{pid};
2765
2766         # fixme: better status?
2767         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2768
2769         my $size = disksize($storecfg, $conf);
2770         if (defined($size)) {
2771             $d->{disk} = 0; # no info available
2772             $d->{maxdisk} = $size;
2773         } else {
2774             $d->{disk} = 0;
2775             $d->{maxdisk} = 0;
2776         }
2777
2778         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2779             * ($conf->{cores} || $defaults->{cores});
2780         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2781         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2782
2783         $d->{name} = $conf->{name} || "VM $vmid";
2784         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2785             : $defaults->{memory}*(1024*1024);
2786
2787         if ($conf->{balloon}) {
2788             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2789             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2790                 : $defaults->{shares};
2791         }
2792
2793         $d->{uptime} = 0;
2794         $d->{cpu} = 0;
2795         $d->{mem} = 0;
2796
2797         $d->{netout} = 0;
2798         $d->{netin} = 0;
2799
2800         $d->{diskread} = 0;
2801         $d->{diskwrite} = 0;
2802
2803         $d->{template} = PVE::QemuConfig->is_template($conf);
2804
2805         $d->{serial} = 1 if conf_has_serial($conf);
2806
2807         $res->{$vmid} = $d;
2808     }
2809
2810     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2811     foreach my $dev (keys %$netdev) {
2812         next if $dev !~ m/^tap([1-9]\d*)i/;
2813         my $vmid = $1;
2814         my $d = $res->{$vmid};
2815         next if !$d;
2816
2817         $d->{netout} += $netdev->{$dev}->{receive};
2818         $d->{netin} += $netdev->{$dev}->{transmit};
2819
2820         if ($full) {
2821             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2822             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2823         }
2824
2825     }
2826
2827     my $ctime = gettimeofday;
2828
2829     foreach my $vmid (keys %$list) {
2830
2831         my $d = $res->{$vmid};
2832         my $pid = $d->{pid};
2833         next if !$pid;
2834
2835         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2836         next if !$pstat; # not running
2837
2838         my $used = $pstat->{utime} + $pstat->{stime};
2839
2840         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2841
2842         if ($pstat->{vsize}) {
2843             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2844         }
2845
2846         my $old = $last_proc_pid_stat->{$pid};
2847         if (!$old) {
2848             $last_proc_pid_stat->{$pid} = {
2849                 time => $ctime,
2850                 used => $used,
2851                 cpu => 0,
2852             };
2853             next;
2854         }
2855
2856         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2857
2858         if ($dtime > 1000) {
2859             my $dutime = $used -  $old->{used};
2860
2861             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2862             $last_proc_pid_stat->{$pid} = {
2863                 time => $ctime,
2864                 used => $used,
2865                 cpu => $d->{cpu},
2866             };
2867         } else {
2868             $d->{cpu} = $old->{cpu};
2869         }
2870     }
2871
2872     return $res if !$full;
2873
2874     my $qmpclient = PVE::QMPClient->new();
2875
2876     my $ballooncb = sub {
2877         my ($vmid, $resp) = @_;
2878
2879         my $info = $resp->{'return'};
2880         return if !$info->{max_mem};
2881
2882         my $d = $res->{$vmid};
2883
2884         # use memory assigned to VM
2885         $d->{maxmem} = $info->{max_mem};
2886         $d->{balloon} = $info->{actual};
2887
2888         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2889             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2890             $d->{freemem} = $info->{free_mem};
2891         }
2892
2893         $d->{ballooninfo} = $info;
2894     };
2895
2896     my $blockstatscb = sub {
2897         my ($vmid, $resp) = @_;
2898         my $data = $resp->{'return'} || [];
2899         my $totalrdbytes = 0;
2900         my $totalwrbytes = 0;
2901
2902         for my $blockstat (@$data) {
2903             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2904             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2905
2906             $blockstat->{device} =~ s/drive-//;
2907             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2908         }
2909         $res->{$vmid}->{diskread} = $totalrdbytes;
2910         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2911     };
2912
2913     my $statuscb = sub {
2914         my ($vmid, $resp) = @_;
2915
2916         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2917         # this fails if ballon driver is not loaded, so this must be
2918         # the last commnand (following command are aborted if this fails).
2919         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2920
2921         my $status = 'unknown';
2922         if (!defined($status = $resp->{'return'}->{status})) {
2923             warn "unable to get VM status\n";
2924             return;
2925         }
2926
2927         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2928     };
2929
2930     foreach my $vmid (keys %$list) {
2931         next if $opt_vmid && ($vmid ne $opt_vmid);
2932         next if !$res->{$vmid}->{pid}; # not running
2933         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2934     }
2935
2936     $qmpclient->queue_execute(undef, 2);
2937
2938     foreach my $vmid (keys %$list) {
2939         next if $opt_vmid && ($vmid ne $opt_vmid);
2940         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2941     }
2942
2943     return $res;
2944 }
2945
2946 sub foreach_drive {
2947     my ($conf, $func, @param) = @_;
2948
2949     foreach my $ds (valid_drive_names()) {
2950         next if !defined($conf->{$ds});
2951
2952         my $drive = parse_drive($ds, $conf->{$ds});
2953         next if !$drive;
2954
2955         &$func($ds, $drive, @param);
2956     }
2957 }
2958
2959 sub foreach_volid {
2960     my ($conf, $func, @param) = @_;
2961
2962     my $volhash = {};
2963
2964     my $test_volid = sub {
2965         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2966
2967         return if !$volid;
2968
2969         $volhash->{$volid}->{cdrom} //= 1;
2970         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2971
2972         $volhash->{$volid}->{replicate} //= 0;
2973         $volhash->{$volid}->{replicate} = 1 if $replicate;
2974
2975         $volhash->{$volid}->{shared} //= 0;
2976         $volhash->{$volid}->{shared} = 1 if $shared;
2977
2978         $volhash->{$volid}->{referenced_in_config} //= 0;
2979         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2980
2981         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2982             if defined($snapname);
2983     };
2984
2985     foreach_drive($conf, sub {
2986         my ($ds, $drive) = @_;
2987         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
2988     });
2989
2990     foreach my $snapname (keys %{$conf->{snapshots}}) {
2991         my $snap = $conf->{snapshots}->{$snapname};
2992         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2993         foreach_drive($snap, sub {
2994             my ($ds, $drive) = @_;
2995             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
2996         });
2997     }
2998
2999     foreach my $volid (keys %$volhash) {
3000         &$func($volid, $volhash->{$volid}, @param);
3001     }
3002 }
3003
3004 sub conf_has_serial {
3005     my ($conf) = @_;
3006
3007     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3008         if ($conf->{"serial$i"}) {
3009             return 1;
3010         }
3011     }
3012
3013     return 0;
3014 }
3015
3016 sub vga_conf_has_spice {
3017     my ($vga) = @_;
3018
3019     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3020
3021     return $1 || 1;
3022 }
3023
3024 sub config_to_command {
3025     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3026
3027     my $cmd = [];
3028     my $globalFlags = [];
3029     my $machineFlags = [];
3030     my $rtcFlags = [];
3031     my $cpuFlags = [];
3032     my $devices = [];
3033     my $pciaddr = '';
3034     my $bridges = {};
3035     my $kvmver = kvm_user_version();
3036     my $vernum = 0; # unknown
3037     my $ostype = $conf->{ostype};
3038     my $winversion = windows_version($ostype);
3039     my $kvm = $conf->{kvm} // 1;
3040
3041     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3042
3043     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3044         $vernum = $1*1000000+$2*1000;
3045     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3046         $vernum = $1*1000000+$2*1000+$3;
3047     }
3048
3049     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3050
3051     my $have_ovz = -f '/proc/vz/vestat';
3052
3053     my $q35 = machine_type_is_q35($conf);
3054     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3055     my $machine_type = $forcemachine || $conf->{machine};
3056     my $use_old_bios_files = undef;
3057     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3058
3059     my $cpuunits = defined($conf->{cpuunits}) ?
3060             $conf->{cpuunits} : $defaults->{cpuunits};
3061
3062     push @$cmd, '/usr/bin/kvm';
3063
3064     push @$cmd, '-id', $vmid;
3065
3066     my $use_virtio = 0;
3067
3068     my $qmpsocket = qmp_socket($vmid);
3069     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3070     push @$cmd, '-mon', "chardev=qmp,mode=control";
3071
3072
3073     push @$cmd, '-pidfile' , pidfile_name($vmid);
3074
3075     push @$cmd, '-daemonize';
3076
3077     if ($conf->{smbios1}) {
3078         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3079     }
3080
3081     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3082         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3083
3084         my $path;
3085         my $format;
3086         if (my $efidisk = $conf->{efidisk0}) {
3087             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3088             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3089             $format = $d->{format};
3090             if ($storeid) {
3091                 $path = PVE::Storage::path($storecfg, $d->{file});
3092                 if (!defined($format)) {
3093                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3094                     $format = qemu_img_format($scfg, $volname);
3095                 }
3096             } else {
3097                 $path = $d->{file};
3098                 die "efidisk format must be specified\n"
3099                     if !defined($format);
3100             }
3101         } else {
3102             warn "no efidisk configured! Using temporary efivars disk.\n";
3103             $path = "/tmp/$vmid-ovmf.fd";
3104             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3105             $format = 'raw';
3106         }
3107
3108         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3109         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3110     }
3111
3112
3113     # add usb controllers
3114     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3115     push @$devices, @usbcontrollers if @usbcontrollers;
3116     my $vga = $conf->{vga};
3117
3118     my $qxlnum = vga_conf_has_spice($vga);
3119     $vga = 'qxl' if $qxlnum;
3120
3121     if (!$vga) {
3122         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3123             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3124         } else {
3125             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3126         }
3127     }
3128
3129     # enable absolute mouse coordinates (needed by vnc)
3130     my $tablet;
3131     if (defined($conf->{tablet})) {
3132         $tablet = $conf->{tablet};
3133     } else {
3134         $tablet = $defaults->{tablet};
3135         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3136         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3137     }
3138
3139     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3140
3141     my $kvm_off = 0;
3142     my $gpu_passthrough;
3143
3144     # host pci devices
3145     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3146         my $d = parse_hostpci($conf->{"hostpci$i"});
3147         next if !$d;
3148
3149         my $pcie = $d->{pcie};
3150         if($pcie){
3151             die "q35 machine model is not enabled" if !$q35;
3152             $pciaddr = print_pcie_addr("hostpci$i");
3153         }else{
3154             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3155         }
3156
3157         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3158         my $romfile = $d->{romfile};
3159
3160         my $xvga = '';
3161         if ($d->{'x-vga'}) {
3162             $xvga = ',x-vga=on';
3163             $kvm_off = 1;
3164             $vga = 'none';
3165             $gpu_passthrough = 1;
3166
3167             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3168                 $xvga = "";
3169             }
3170         }
3171         my $pcidevices = $d->{pciid};
3172         my $multifunction = 1 if @$pcidevices > 1;
3173
3174         my $j=0;
3175         foreach my $pcidevice (@$pcidevices) {
3176
3177             my $id = "hostpci$i";
3178             $id .= ".$j" if $multifunction;
3179             my $addr = $pciaddr;
3180             $addr .= ".$j" if $multifunction;
3181             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3182
3183             if($j == 0){
3184                 $devicestr .= "$rombar$xvga";
3185                 $devicestr .= ",multifunction=on" if $multifunction;
3186                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3187             }
3188
3189             push @$devices, '-device', $devicestr;
3190             $j++;
3191         }
3192     }
3193
3194     # usb devices
3195     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3196     push @$devices, @usbdevices if @usbdevices;
3197     # serial devices
3198     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3199         if (my $path = $conf->{"serial$i"}) {
3200             if ($path eq 'socket') {
3201                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3202                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3203                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3204             } else {
3205                 die "no such serial device\n" if ! -c $path;
3206                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3207                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3208             }
3209         }
3210     }
3211
3212     # parallel devices
3213     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3214         if (my $path = $conf->{"parallel$i"}) {
3215             die "no such parallel device\n" if ! -c $path;
3216             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3217             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3218             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3219         }
3220     }
3221
3222     my $vmname = $conf->{name} || "vm$vmid";
3223
3224     push @$cmd, '-name', $vmname;
3225
3226     my $sockets = 1;
3227     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3228     $sockets = $conf->{sockets} if  $conf->{sockets};
3229
3230     my $cores = $conf->{cores} || 1;
3231
3232     my $maxcpus = $sockets * $cores;
3233
3234     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3235
3236     my $allowed_vcpus = $cpuinfo->{cpus};
3237
3238     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3239         if ($allowed_vcpus < $maxcpus);
3240
3241     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3242
3243         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3244         for (my $i = 2; $i <= $vcpus; $i++)  {
3245             my $cpustr = print_cpu_device($conf,$i);
3246             push @$cmd, '-device', $cpustr;
3247         }
3248
3249     } else {
3250
3251         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3252     }
3253     push @$cmd, '-nodefaults';
3254
3255     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3256
3257     my $bootindex_hash = {};
3258     my $i = 1;
3259     foreach my $o (split(//, $bootorder)) {
3260         $bootindex_hash->{$o} = $i*100;
3261         $i++;
3262     }
3263
3264     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3265
3266     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3267
3268     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3269
3270     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3271
3272     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3273         my $socket = vnc_socket($vmid);
3274         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3275     } else {
3276         push @$cmd, '-nographic';
3277     }
3278
3279     # time drift fix
3280     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3281
3282     my $useLocaltime = $conf->{localtime};
3283
3284     if ($winversion >= 5) { # windows
3285         $useLocaltime = 1 if !defined($conf->{localtime});
3286
3287         # use time drift fix when acpi is enabled
3288         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3289             $tdf = 1 if !defined($conf->{tdf});
3290         }
3291     }
3292
3293     if ($winversion >= 6) {
3294         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3295         push @$cmd, '-no-hpet';
3296     }
3297
3298     push @$rtcFlags, 'driftfix=slew' if $tdf;
3299
3300     if (!$kvm) {
3301         push @$machineFlags, 'accel=tcg';
3302     }
3303
3304     if ($machine_type) {
3305         push @$machineFlags, "type=${machine_type}";
3306     }
3307
3308     if ($conf->{startdate}) {
3309         push @$rtcFlags, "base=$conf->{startdate}";
3310     } elsif ($useLocaltime) {
3311         push @$rtcFlags, 'base=localtime';
3312     }
3313
3314     my $cpu = $kvm ? "kvm64" : "qemu64";
3315     if (my $cputype = $conf->{cpu}) {
3316         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3317             or die "Cannot parse cpu description: $cputype\n";
3318         $cpu = $cpuconf->{cputype};
3319         $kvm_off = 1 if $cpuconf->{hidden};
3320
3321         if (defined(my $flags = $cpuconf->{flags})) {
3322             push @$cpuFlags, split(";", $flags);
3323         }
3324     }
3325
3326     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3327
3328     push @$cpuFlags , '-x2apic'
3329         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3330
3331     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3332
3333     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3334
3335     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3336
3337         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3338         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3339     }
3340
3341     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3342
3343     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3344
3345     push @$cpuFlags, 'kvm=off' if $kvm_off;
3346
3347     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3348         die "internal error"; # should not happen
3349
3350     push @$cpuFlags, "vendor=${cpu_vendor}"
3351         if $cpu_vendor ne 'default';
3352
3353     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3354
3355     push @$cmd, '-cpu', $cpu;
3356
3357     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3358
3359     push @$cmd, '-S' if $conf->{freeze};
3360
3361     # set keyboard layout
3362     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3363     push @$cmd, '-k', $kb if $kb;
3364
3365     # enable sound
3366     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3367     #push @$cmd, '-soundhw', 'es1370';
3368     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3369
3370     if($conf->{agent}) {
3371         my $qgasocket = qmp_socket($vmid, 1);
3372         my $pciaddr = print_pci_addr("qga0", $bridges);
3373         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3374         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3375         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3376     }
3377
3378     my $spice_port;
3379
3380     if ($qxlnum) {
3381         if ($qxlnum > 1) {
3382             if ($winversion){
3383                 for(my $i = 1; $i < $qxlnum; $i++){
3384                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3385                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3386                 }
3387             } else {
3388                 # assume other OS works like Linux
3389                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3390                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3391             }
3392         }
3393
3394         my $pciaddr = print_pci_addr("spice", $bridges);
3395
3396         my $nodename = PVE::INotify::nodename();
3397         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3398         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3399         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3400         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3401         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3402
3403         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3404
3405         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3406         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3407         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3408     }
3409
3410     # enable balloon by default, unless explicitly disabled
3411     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3412         $pciaddr = print_pci_addr("balloon0", $bridges);
3413         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3414     }
3415
3416     if ($conf->{watchdog}) {
3417         my $wdopts = parse_watchdog($conf->{watchdog});
3418         $pciaddr = print_pci_addr("watchdog", $bridges);
3419         my $watchdog = $wdopts->{model} || 'i6300esb';
3420         push @$devices, '-device', "$watchdog$pciaddr";
3421         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3422     }
3423
3424     my $vollist = [];
3425     my $scsicontroller = {};
3426     my $ahcicontroller = {};
3427     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3428
3429     # Add iscsi initiator name if available
3430     if (my $initiator = get_initiator_name()) {
3431         push @$devices, '-iscsi', "initiator-name=$initiator";
3432     }
3433
3434     foreach_drive($conf, sub {
3435         my ($ds, $drive) = @_;
3436
3437         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3438             push @$vollist, $drive->{file};
3439         }
3440
3441         # ignore efidisk here, already added in bios/fw handling code above
3442         return if $drive->{interface} eq 'efidisk';
3443
3444         $use_virtio = 1 if $ds =~ m/^virtio/;
3445
3446         if (drive_is_cdrom ($drive)) {
3447             if ($bootindex_hash->{d}) {
3448                 $drive->{bootindex} = $bootindex_hash->{d};
3449                 $bootindex_hash->{d} += 1;
3450             }
3451         } else {
3452             if ($bootindex_hash->{c}) {
3453                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3454                 $bootindex_hash->{c} += 1;
3455             }
3456         }
3457
3458         if($drive->{interface} eq 'virtio'){
3459            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3460         }
3461
3462         if ($drive->{interface} eq 'scsi') {
3463
3464             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3465
3466             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3467             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3468
3469             my $iothread = '';
3470             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3471                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3472                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3473             } elsif ($drive->{iothread}) {
3474                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3475             }
3476
3477             my $queues = '';
3478             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3479                 $queues = ",num_queues=$drive->{queues}";
3480             }
3481
3482             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3483             $scsicontroller->{$controller}=1;
3484         }
3485
3486         if ($drive->{interface} eq 'sata') {
3487            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3488            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3489            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3490            $ahcicontroller->{$controller}=1;
3491         }
3492
3493         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3494         push @$devices, '-drive',$drive_cmd;
3495         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3496     });
3497
3498     for (my $i = 0; $i < $MAX_NETS; $i++) {
3499          next if !$conf->{"net$i"};
3500          my $d = parse_net($conf->{"net$i"});
3501          next if !$d;
3502
3503          $use_virtio = 1 if $d->{model} eq 'virtio';
3504
3505          if ($bootindex_hash->{n}) {
3506             $d->{bootindex} = $bootindex_hash->{n};
3507             $bootindex_hash->{n} += 1;
3508          }
3509
3510          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3511          push @$devices, '-netdev', $netdevfull;
3512
3513          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3514          push @$devices, '-device', $netdevicefull;
3515     }
3516
3517     if (!$q35) {
3518         # add pci bridges
3519         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3520            $bridges->{1} = 1;
3521            $bridges->{2} = 1;
3522         }
3523
3524         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3525
3526         while (my ($k, $v) = each %$bridges) {
3527             $pciaddr = print_pci_addr("pci.$k");
3528             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3529         }
3530     }
3531
3532     # add custom args
3533     if ($conf->{args}) {
3534         my $aa = PVE::Tools::split_args($conf->{args});
3535         push @$cmd, @$aa;
3536     }
3537
3538     push @$cmd, @$devices;
3539     push @$cmd, '-rtc', join(',', @$rtcFlags)
3540         if scalar(@$rtcFlags);
3541     push @$cmd, '-machine', join(',', @$machineFlags)
3542         if scalar(@$machineFlags);
3543     push @$cmd, '-global', join(',', @$globalFlags)
3544         if scalar(@$globalFlags);
3545
3546     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3547 }
3548
3549 sub vnc_socket {
3550     my ($vmid) = @_;
3551     return "${var_run_tmpdir}/$vmid.vnc";
3552 }
3553
3554 sub spice_port {
3555     my ($vmid) = @_;
3556
3557     my $res = vm_mon_cmd($vmid, 'query-spice');
3558
3559     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3560 }
3561
3562 sub qmp_socket {
3563     my ($vmid, $qga) = @_;
3564     my $sockettype = $qga ? 'qga' : 'qmp';
3565     return "${var_run_tmpdir}/$vmid.$sockettype";
3566 }
3567
3568 sub pidfile_name {
3569     my ($vmid) = @_;
3570     return "${var_run_tmpdir}/$vmid.pid";
3571 }
3572
3573 sub vm_devices_list {
3574     my ($vmid) = @_;
3575
3576     my $res = vm_mon_cmd($vmid, 'query-pci');
3577     my $devices = {};
3578     foreach my $pcibus (@$res) {
3579         foreach my $device (@{$pcibus->{devices}}) {
3580             next if !$device->{'qdev_id'};
3581             if ($device->{'pci_bridge'}) {
3582                 $devices->{$device->{'qdev_id'}} = 1;
3583                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3584                     next if !$bridge_device->{'qdev_id'};
3585                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3586                     $devices->{$device->{'qdev_id'}}++;
3587                 }
3588             } else {
3589                 $devices->{$device->{'qdev_id'}} = 1;
3590             }
3591         }
3592     }
3593
3594     my $resblock = vm_mon_cmd($vmid, 'query-block');
3595     foreach my $block (@$resblock) {
3596         if($block->{device} =~ m/^drive-(\S+)/){
3597                 $devices->{$1} = 1;
3598         }
3599     }
3600
3601     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3602     foreach my $mice (@$resmice) {
3603         if ($mice->{name} eq 'QEMU HID Tablet') {
3604             $devices->{tablet} = 1;
3605             last;
3606         }
3607     }
3608
3609     # for usb devices there is no query-usb
3610     # but we can iterate over the entries in
3611     # qom-list path=/machine/peripheral
3612     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3613     foreach my $per (@$resperipheral) {
3614         if ($per->{name} =~ m/^usb\d+$/) {
3615             $devices->{$per->{name}} = 1;
3616         }
3617     }
3618
3619     return $devices;
3620 }
3621
3622 sub vm_deviceplug {
3623     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3624
3625     my $q35 = machine_type_is_q35($conf);
3626
3627     my $devices_list = vm_devices_list($vmid);
3628     return 1 if defined($devices_list->{$deviceid});
3629
3630     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3631
3632     if ($deviceid eq 'tablet') {
3633
3634         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3635
3636     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3637
3638         die "usb hotplug currently not reliable\n";
3639         # since we can't reliably hot unplug all added usb devices
3640         # and usb passthrough disables live migration
3641         # we disable usb hotplugging for now
3642         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3643
3644     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3645
3646         qemu_iothread_add($vmid, $deviceid, $device);
3647
3648         qemu_driveadd($storecfg, $vmid, $device);
3649         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3650
3651         qemu_deviceadd($vmid, $devicefull);
3652         eval { qemu_deviceaddverify($vmid, $deviceid); };
3653         if (my $err = $@) {
3654             eval { qemu_drivedel($vmid, $deviceid); };
3655             warn $@ if $@;
3656             die $err;
3657         }
3658
3659     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3660
3661
3662         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3663         my $pciaddr = print_pci_addr($deviceid);
3664         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3665
3666         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3667
3668         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3669             qemu_iothread_add($vmid, $deviceid, $device);
3670             $devicefull .= ",iothread=iothread-$deviceid";
3671         }
3672
3673         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3674             $devicefull .= ",num_queues=$device->{queues}";
3675         }
3676
3677         qemu_deviceadd($vmid, $devicefull);
3678         qemu_deviceaddverify($vmid, $deviceid);
3679
3680     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3681
3682         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3683         qemu_driveadd($storecfg, $vmid, $device);
3684
3685         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3686         eval { qemu_deviceadd($vmid, $devicefull); };
3687         if (my $err = $@) {
3688             eval { qemu_drivedel($vmid, $deviceid); };
3689             warn $@ if $@;
3690             die $err;
3691         }
3692
3693     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3694
3695         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3696
3697         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3698         my $use_old_bios_files = undef;
3699         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3700
3701         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3702         qemu_deviceadd($vmid, $netdevicefull);
3703         eval { qemu_deviceaddverify($vmid, $deviceid); };
3704         if (my $err = $@) {
3705             eval { qemu_netdevdel($vmid, $deviceid); };
3706             warn $@ if $@;
3707             die $err;
3708         }
3709
3710     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3711
3712         my $bridgeid = $2;
3713         my $pciaddr = print_pci_addr($deviceid);
3714         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3715
3716         qemu_deviceadd($vmid, $devicefull);
3717         qemu_deviceaddverify($vmid, $deviceid);
3718
3719     } else {
3720         die "can't hotplug device '$deviceid'\n";
3721     }
3722
3723     return 1;
3724 }
3725
3726 # fixme: this should raise exceptions on error!
3727 sub vm_deviceunplug {
3728     my ($vmid, $conf, $deviceid) = @_;
3729
3730     my $devices_list = vm_devices_list($vmid);
3731     return 1 if !defined($devices_list->{$deviceid});
3732
3733     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3734
3735     if ($deviceid eq 'tablet') {
3736
3737         qemu_devicedel($vmid, $deviceid);
3738
3739     } elsif ($deviceid =~ m/^usb\d+$/) {
3740
3741         die "usb hotplug currently not reliable\n";
3742         # when unplugging usb devices this way,
3743         # there may be remaining usb controllers/hubs
3744         # so we disable it for now
3745         qemu_devicedel($vmid, $deviceid);
3746         qemu_devicedelverify($vmid, $deviceid);
3747
3748     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3749
3750         qemu_devicedel($vmid, $deviceid);
3751         qemu_devicedelverify($vmid, $deviceid);
3752         qemu_drivedel($vmid, $deviceid);
3753         qemu_iothread_del($conf, $vmid, $deviceid);
3754
3755     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3756
3757         qemu_devicedel($vmid, $deviceid);
3758         qemu_devicedelverify($vmid, $deviceid);
3759         qemu_iothread_del($conf, $vmid, $deviceid);
3760
3761     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3762
3763         qemu_devicedel($vmid, $deviceid);
3764         qemu_drivedel($vmid, $deviceid);
3765         qemu_deletescsihw($conf, $vmid, $deviceid);
3766
3767     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3768
3769         qemu_devicedel($vmid, $deviceid);
3770         qemu_devicedelverify($vmid, $deviceid);
3771         qemu_netdevdel($vmid, $deviceid);
3772
3773     } else {
3774         die "can't unplug device '$deviceid'\n";
3775     }
3776
3777     return 1;
3778 }
3779
3780 sub qemu_deviceadd {
3781     my ($vmid, $devicefull) = @_;
3782
3783     $devicefull = "driver=".$devicefull;
3784     my %options =  split(/[=,]/, $devicefull);
3785
3786     vm_mon_cmd($vmid, "device_add" , %options);
3787 }
3788
3789 sub qemu_devicedel {
3790     my ($vmid, $deviceid) = @_;
3791
3792     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3793 }
3794
3795 sub qemu_iothread_add {
3796     my($vmid, $deviceid, $device) = @_;
3797
3798     if ($device->{iothread}) {
3799         my $iothreads = vm_iothreads_list($vmid);
3800         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3801     }
3802 }
3803
3804 sub qemu_iothread_del {
3805     my($conf, $vmid, $deviceid) = @_;
3806
3807     my $device = parse_drive($deviceid, $conf->{$deviceid});
3808     if ($device->{iothread}) {
3809         my $iothreads = vm_iothreads_list($vmid);
3810         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3811     }
3812 }
3813
3814 sub qemu_objectadd {
3815     my($vmid, $objectid, $qomtype) = @_;
3816
3817     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3818
3819     return 1;
3820 }
3821
3822 sub qemu_objectdel {
3823     my($vmid, $objectid) = @_;
3824
3825     vm_mon_cmd($vmid, "object-del", id => $objectid);
3826
3827     return 1;
3828 }
3829
3830 sub qemu_driveadd {
3831     my ($storecfg, $vmid, $device) = @_;
3832
3833     my $drive = print_drive_full($storecfg, $vmid, $device);
3834     $drive =~ s/\\/\\\\/g;
3835     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3836
3837     # If the command succeeds qemu prints: "OK"
3838     return 1 if $ret =~ m/OK/s;
3839
3840     die "adding drive failed: $ret\n";
3841 }
3842
3843 sub qemu_drivedel {
3844     my($vmid, $deviceid) = @_;
3845
3846     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3847     $ret =~ s/^\s+//;
3848
3849     return 1 if $ret eq "";
3850
3851     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3852     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3853
3854     die "deleting drive $deviceid failed : $ret\n";
3855 }
3856
3857 sub qemu_deviceaddverify {
3858     my ($vmid, $deviceid) = @_;
3859
3860     for (my $i = 0; $i <= 5; $i++) {
3861          my $devices_list = vm_devices_list($vmid);
3862          return 1 if defined($devices_list->{$deviceid});
3863          sleep 1;
3864     }
3865
3866     die "error on hotplug device '$deviceid'\n";
3867 }
3868
3869
3870 sub qemu_devicedelverify {
3871     my ($vmid, $deviceid) = @_;
3872
3873     # need to verify that the device is correctly removed as device_del
3874     # is async and empty return is not reliable
3875
3876     for (my $i = 0; $i <= 5; $i++) {
3877          my $devices_list = vm_devices_list($vmid);
3878          return 1 if !defined($devices_list->{$deviceid});
3879          sleep 1;
3880     }
3881
3882     die "error on hot-unplugging device '$deviceid'\n";
3883 }
3884
3885 sub qemu_findorcreatescsihw {
3886     my ($storecfg, $conf, $vmid, $device) = @_;
3887
3888     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3889
3890     my $scsihwid="$controller_prefix$controller";
3891     my $devices_list = vm_devices_list($vmid);
3892
3893     if(!defined($devices_list->{$scsihwid})) {
3894         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3895     }
3896
3897     return 1;
3898 }
3899
3900 sub qemu_deletescsihw {
3901     my ($conf, $vmid, $opt) = @_;
3902
3903     my $device = parse_drive($opt, $conf->{$opt});
3904
3905     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3906         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3907         return 1;
3908     }
3909
3910     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3911
3912     my $devices_list = vm_devices_list($vmid);
3913     foreach my $opt (keys %{$devices_list}) {
3914         if (PVE::QemuServer::is_valid_drivename($opt)) {
3915             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3916             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3917                 return 1;
3918             }
3919         }
3920     }
3921
3922     my $scsihwid="scsihw$controller";
3923
3924     vm_deviceunplug($vmid, $conf, $scsihwid);
3925
3926     return 1;
3927 }
3928
3929 sub qemu_add_pci_bridge {
3930     my ($storecfg, $conf, $vmid, $device) = @_;
3931
3932     my $bridges = {};
3933
3934     my $bridgeid;
3935
3936     print_pci_addr($device, $bridges);
3937
3938     while (my ($k, $v) = each %$bridges) {
3939         $bridgeid = $k;
3940     }
3941     return 1 if !defined($bridgeid) || $bridgeid < 1;
3942
3943     my $bridge = "pci.$bridgeid";
3944     my $devices_list = vm_devices_list($vmid);
3945
3946     if (!defined($devices_list->{$bridge})) {
3947         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3948     }
3949
3950     return 1;
3951 }
3952
3953 sub qemu_set_link_status {
3954     my ($vmid, $device, $up) = @_;
3955
3956     vm_mon_cmd($vmid, "set_link", name => $device,
3957                up => $up ? JSON::true : JSON::false);
3958 }
3959
3960 sub qemu_netdevadd {
3961     my ($vmid, $conf, $device, $deviceid) = @_;
3962
3963     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3964     my %options =  split(/[=,]/, $netdev);
3965
3966     vm_mon_cmd($vmid, "netdev_add",  %options);
3967     return 1;
3968 }
3969
3970 sub qemu_netdevdel {
3971     my ($vmid, $deviceid) = @_;
3972
3973     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3974 }
3975
3976 sub qemu_usb_hotplug {
3977     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3978
3979     return if !$device;
3980
3981     # remove the old one first
3982     vm_deviceunplug($vmid, $conf, $deviceid);
3983
3984     # check if xhci controller is necessary and available
3985     if ($device->{usb3}) {
3986
3987         my $devicelist = vm_devices_list($vmid);
3988
3989         if (!$devicelist->{xhci}) {
3990             my $pciaddr = print_pci_addr("xhci");
3991             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3992         }
3993     }
3994     my $d = parse_usb_device($device->{host});
3995     $d->{usb3} = $device->{usb3};
3996
3997     # add the new one
3998     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3999 }
4000
4001 sub qemu_cpu_hotplug {
4002     my ($vmid, $conf, $vcpus) = @_;
4003
4004     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4005
4006     my $sockets = 1;
4007     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4008     $sockets = $conf->{sockets} if  $conf->{sockets};
4009     my $cores = $conf->{cores} || 1;
4010     my $maxcpus = $sockets * $cores;
4011
4012     $vcpus = $maxcpus if !$vcpus;
4013
4014     die "you can't add more vcpus than maxcpus\n"
4015         if $vcpus > $maxcpus;
4016
4017     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4018
4019     if ($vcpus < $currentvcpus) {
4020
4021         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4022
4023             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4024                 qemu_devicedel($vmid, "cpu$i");
4025                 my $retry = 0;
4026                 my $currentrunningvcpus = undef;
4027                 while (1) {
4028                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4029                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4030                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4031                     $retry++;
4032                     sleep 1;
4033                 }
4034                 #update conf after each succesfull cpu unplug
4035                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4036                 PVE::QemuConfig->write_config($vmid, $conf);
4037             }
4038         } else {
4039             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4040         }
4041
4042         return;
4043     }
4044
4045     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4046     die "vcpus in running vm does not match its configuration\n"
4047         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4048
4049     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4050
4051         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4052             my $cpustr = print_cpu_device($conf, $i);
4053             qemu_deviceadd($vmid, $cpustr);
4054
4055             my $retry = 0;
4056             my $currentrunningvcpus = undef;
4057             while (1) {
4058                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4059                 last if scalar(@{$currentrunningvcpus}) == $i;
4060                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4061                 sleep 1;
4062                 $retry++;
4063             }
4064             #update conf after each succesfull cpu hotplug
4065             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4066             PVE::QemuConfig->write_config($vmid, $conf);
4067         }
4068     } else {
4069
4070         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4071             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4072         }
4073     }
4074 }
4075
4076 sub qemu_block_set_io_throttle {
4077     my ($vmid, $deviceid,
4078         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4079         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4080         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4081         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4082
4083     return if !check_running($vmid) ;
4084
4085     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4086         bps => int($bps),
4087         bps_rd => int($bps_rd),
4088         bps_wr => int($bps_wr),
4089         iops => int($iops),
4090         iops_rd => int($iops_rd),
4091         iops_wr => int($iops_wr),
4092         bps_max => int($bps_max),
4093         bps_rd_max => int($bps_rd_max),
4094         bps_wr_max => int($bps_wr_max),
4095         iops_max => int($iops_max),
4096         iops_rd_max => int($iops_rd_max),
4097         iops_wr_max => int($iops_wr_max),
4098         bps_max_length => int($bps_max_length),
4099         bps_rd_max_length => int($bps_rd_max_length),
4100         bps_wr_max_length => int($bps_wr_max_length),
4101         iops_max_length => int($iops_max_length),
4102         iops_rd_max_length => int($iops_rd_max_length),
4103         iops_wr_max_length => int($iops_wr_max_length),
4104     );
4105
4106 }
4107
4108 # old code, only used to shutdown old VM after update
4109 sub __read_avail {
4110     my ($fh, $timeout) = @_;
4111
4112     my $sel = new IO::Select;
4113     $sel->add($fh);
4114
4115     my $res = '';
4116     my $buf;
4117
4118     my @ready;
4119     while (scalar (@ready = $sel->can_read($timeout))) {
4120         my $count;
4121         if ($count = $fh->sysread($buf, 8192)) {
4122             if ($buf =~ /^(.*)\(qemu\) $/s) {
4123                 $res .= $1;
4124                 last;
4125             } else {
4126                 $res .= $buf;
4127             }
4128         } else {
4129             if (!defined($count)) {
4130                 die "$!\n";
4131             }
4132             last;
4133         }
4134     }
4135
4136     die "monitor read timeout\n" if !scalar(@ready);
4137
4138     return $res;
4139 }
4140
4141 # old code, only used to shutdown old VM after update
4142 sub vm_monitor_command {
4143     my ($vmid, $cmdstr, $nocheck) = @_;
4144
4145     my $res;
4146
4147     eval {
4148         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4149
4150         my $sname = "${var_run_tmpdir}/$vmid.mon";
4151
4152         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4153             die "unable to connect to VM $vmid socket - $!\n";
4154
4155         my $timeout = 3;
4156
4157         # hack: migrate sometime blocks the monitor (when migrate_downtime
4158         # is set)
4159         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4160             $timeout = 60*60; # 1 hour
4161         }
4162
4163         # read banner;
4164         my $data = __read_avail($sock, $timeout);
4165
4166         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4167             die "got unexpected qemu monitor banner\n";
4168         }
4169
4170         my $sel = new IO::Select;
4171         $sel->add($sock);
4172
4173         if (!scalar(my @ready = $sel->can_write($timeout))) {
4174             die "monitor write error - timeout";
4175         }
4176
4177         my $fullcmd = "$cmdstr\r";
4178
4179         # syslog('info', "VM $vmid monitor command: $cmdstr");
4180
4181         my $b;
4182         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4183             die "monitor write error - $!";
4184         }
4185
4186         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4187
4188         $timeout = 20;
4189
4190         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4191             $timeout = 60*60; # 1 hour
4192         } elsif ($cmdstr =~ m/^(eject|change)/) {
4193             $timeout = 60; # note: cdrom mount command is slow
4194         }
4195         if ($res = __read_avail($sock, $timeout)) {
4196
4197             my @lines = split("\r?\n", $res);
4198
4199             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4200
4201             $res = join("\n", @lines);
4202             $res .= "\n";
4203         }
4204     };
4205
4206     my $err = $@;
4207
4208     if ($err) {
4209         syslog("err", "VM $vmid monitor command failed - $err");
4210         die $err;
4211     }
4212
4213     return $res;
4214 }
4215
4216 sub qemu_block_resize {
4217     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4218
4219     my $running = check_running($vmid);
4220
4221     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4222
4223     return if !$running;
4224
4225     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4226
4227 }
4228
4229 sub qemu_volume_snapshot {
4230     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4231
4232     my $running = check_running($vmid);
4233
4234     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4235         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4236     } else {
4237         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4238     }
4239 }
4240
4241 sub qemu_volume_snapshot_delete {
4242     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4243
4244     my $running = check_running($vmid);
4245
4246     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4247         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4248     } else {
4249         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4250     }
4251 }
4252
4253 sub set_migration_caps {
4254     my ($vmid) = @_;
4255
4256     my $cap_ref = [];
4257
4258     my $enabled_cap = {
4259         "auto-converge" => 1,
4260         "xbzrle" => 1,
4261         "x-rdma-pin-all" => 0,
4262         "zero-blocks" => 0,
4263         "compress" => 0
4264     };
4265
4266     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4267
4268     for my $supported_capability (@$supported_capabilities) {
4269         push @$cap_ref, {
4270             capability => $supported_capability->{capability},
4271             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4272         };
4273     }
4274
4275     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4276 }
4277
4278 my $fast_plug_option = {
4279     'lock' => 1,
4280     'name' => 1,
4281     'onboot' => 1,
4282     'shares' => 1,
4283     'startup' => 1,
4284     'description' => 1,
4285     'protection' => 1,
4286     'vmstatestorage' => 1,
4287 };
4288
4289 # hotplug changes in [PENDING]
4290 # $selection hash can be used to only apply specified options, for
4291 # example: { cores => 1 } (only apply changed 'cores')
4292 # $errors ref is used to return error messages
4293 sub vmconfig_hotplug_pending {
4294     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4295
4296     my $defaults = load_defaults();
4297
4298     # commit values which do not have any impact on running VM first
4299     # Note: those option cannot raise errors, we we do not care about
4300     # $selection and always apply them.
4301
4302     my $add_error = sub {
4303         my ($opt, $msg) = @_;
4304         $errors->{$opt} = "hotplug problem - $msg";
4305     };
4306
4307     my $changes = 0;
4308     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4309         if ($fast_plug_option->{$opt}) {
4310             $conf->{$opt} = $conf->{pending}->{$opt};
4311             delete $conf->{pending}->{$opt};
4312             $changes = 1;
4313         }
4314     }
4315
4316     if ($changes) {
4317         PVE::QemuConfig->write_config($vmid, $conf);
4318         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4319     }
4320
4321     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4322
4323     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4324     while (my ($opt, $force) = each %$pending_delete_hash) {
4325         next if $selection && !$selection->{$opt};
4326         eval {
4327             if ($opt eq 'hotplug') {
4328                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4329             } elsif ($opt eq 'tablet') {
4330                 die "skip\n" if !$hotplug_features->{usb};
4331                 if ($defaults->{tablet}) {
4332                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4333                 } else {
4334                     vm_deviceunplug($vmid, $conf, $opt);
4335                 }
4336             } elsif ($opt =~ m/^usb\d+/) {
4337                 die "skip\n";
4338                 # since we cannot reliably hot unplug usb devices
4339                 # we are disabling it
4340                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4341                 vm_deviceunplug($vmid, $conf, $opt);
4342             } elsif ($opt eq 'vcpus') {
4343                 die "skip\n" if !$hotplug_features->{cpu};
4344                 qemu_cpu_hotplug($vmid, $conf, undef);
4345             } elsif ($opt eq 'balloon') {
4346                 # enable balloon device is not hotpluggable
4347                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4348             } elsif ($fast_plug_option->{$opt}) {
4349                 # do nothing
4350             } elsif ($opt =~ m/^net(\d+)$/) {
4351                 die "skip\n" if !$hotplug_features->{network};
4352                 vm_deviceunplug($vmid, $conf, $opt);
4353             } elsif (is_valid_drivename($opt)) {
4354                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4355                 vm_deviceunplug($vmid, $conf, $opt);
4356                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4357             } elsif ($opt =~ m/^memory$/) {
4358                 die "skip\n" if !$hotplug_features->{memory};
4359                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4360             } elsif ($opt eq 'cpuunits') {
4361                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4362             } elsif ($opt eq 'cpulimit') {
4363                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4364             } else {
4365                 die "skip\n";
4366             }
4367         };
4368         if (my $err = $@) {
4369             &$add_error($opt, $err) if $err ne "skip\n";
4370         } else {
4371             # save new config if hotplug was successful
4372             delete $conf->{$opt};
4373             vmconfig_undelete_pending_option($conf, $opt);
4374             PVE::QemuConfig->write_config($vmid, $conf);
4375             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4376         }
4377     }
4378
4379     foreach my $opt (keys %{$conf->{pending}}) {
4380         next if $selection && !$selection->{$opt};
4381         my $value = $conf->{pending}->{$opt};
4382         eval {
4383             if ($opt eq 'hotplug') {
4384                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4385             } elsif ($opt eq 'tablet') {
4386                 die "skip\n" if !$hotplug_features->{usb};
4387                 if ($value == 1) {
4388                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4389                 } elsif ($value == 0) {
4390                     vm_deviceunplug($vmid, $conf, $opt);
4391                 }
4392             } elsif ($opt =~ m/^usb\d+$/) {
4393                 die "skip\n";
4394                 # since we cannot reliably hot unplug usb devices
4395                 # we are disabling it
4396                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4397                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4398                 die "skip\n" if !$d;
4399                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4400             } elsif ($opt eq 'vcpus') {
4401                 die "skip\n" if !$hotplug_features->{cpu};
4402                 qemu_cpu_hotplug($vmid, $conf, $value);
4403             } elsif ($opt eq 'balloon') {
4404                 # enable/disable balloning device is not hotpluggable
4405                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4406                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4407                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4408
4409                 # allow manual ballooning if shares is set to zero
4410                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4411                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4412                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4413                 }
4414             } elsif ($opt =~ m/^net(\d+)$/) {
4415                 # some changes can be done without hotplug
4416                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4417                                     $vmid, $opt, $value);
4418             } elsif (is_valid_drivename($opt)) {
4419                 # some changes can be done without hotplug
4420                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4421                                      $vmid, $opt, $value, 1);
4422             } elsif ($opt =~ m/^memory$/) { #dimms
4423                 die "skip\n" if !$hotplug_features->{memory};
4424                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4425             } elsif ($opt eq 'cpuunits') {
4426                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4427             } elsif ($opt eq 'cpulimit') {
4428                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4429                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4430             } else {
4431                 die "skip\n";  # skip non-hot-pluggable options
4432             }
4433         };
4434         if (my $err = $@) {
4435             &$add_error($opt, $err) if $err ne "skip\n";
4436         } else {
4437             # save new config if hotplug was successful
4438             $conf->{$opt} = $value;
4439             delete $conf->{pending}->{$opt};
4440             PVE::QemuConfig->write_config($vmid, $conf);
4441             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4442         }
4443     }
4444 }
4445
4446 sub try_deallocate_drive {
4447     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4448
4449     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4450         my $volid = $drive->{file};
4451         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4452             my $sid = PVE::Storage::parse_volume_id($volid);
4453             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4454
4455             # check if the disk is really unused
4456             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4457                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4458             PVE::Storage::vdisk_free($storecfg, $volid);
4459             return 1;
4460         } else {
4461             # If vm is not owner of this disk remove from config
4462             return 1;
4463         }
4464     }
4465
4466     return undef;
4467 }
4468
4469 sub vmconfig_delete_or_detach_drive {
4470     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4471
4472     my $drive = parse_drive($opt, $conf->{$opt});
4473
4474     my $rpcenv = PVE::RPCEnvironment::get();
4475     my $authuser = $rpcenv->get_user();
4476
4477     if ($force) {
4478         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4479         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4480     } else {
4481         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4482     }
4483 }
4484
4485 sub vmconfig_apply_pending {
4486     my ($vmid, $conf, $storecfg) = @_;
4487
4488     # cold plug
4489
4490     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4491     while (my ($opt, $force) = each %$pending_delete_hash) {
4492         die "internal error" if $opt =~ m/^unused/;
4493         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4494         if (!defined($conf->{$opt})) {
4495             vmconfig_undelete_pending_option($conf, $opt);
4496             PVE::QemuConfig->write_config($vmid, $conf);
4497         } elsif (is_valid_drivename($opt)) {
4498             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4499             vmconfig_undelete_pending_option($conf, $opt);
4500             delete $conf->{$opt};
4501             PVE::QemuConfig->write_config($vmid, $conf);
4502         } else {
4503             vmconfig_undelete_pending_option($conf, $opt);
4504             delete $conf->{$opt};
4505             PVE::QemuConfig->write_config($vmid, $conf);
4506         }
4507     }
4508
4509     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4510
4511     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4512         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4513
4514         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4515             # skip if nothing changed
4516         } elsif (is_valid_drivename($opt)) {
4517             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4518                 if defined($conf->{$opt});
4519             $conf->{$opt} = $conf->{pending}->{$opt};
4520         } else {
4521             $conf->{$opt} = $conf->{pending}->{$opt};
4522         }
4523
4524         delete $conf->{pending}->{$opt};
4525         PVE::QemuConfig->write_config($vmid, $conf);
4526     }
4527 }
4528
4529 my $safe_num_ne = sub {
4530     my ($a, $b) = @_;
4531
4532     return 0 if !defined($a) && !defined($b);
4533     return 1 if !defined($a);
4534     return 1 if !defined($b);
4535
4536     return $a != $b;
4537 };
4538
4539 my $safe_string_ne = sub {
4540     my ($a, $b) = @_;
4541
4542     return 0 if !defined($a) && !defined($b);
4543     return 1 if !defined($a);
4544     return 1 if !defined($b);
4545
4546     return $a ne $b;
4547 };
4548
4549 sub vmconfig_update_net {
4550     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4551
4552     my $newnet = parse_net($value);
4553
4554     if ($conf->{$opt}) {
4555         my $oldnet = parse_net($conf->{$opt});
4556
4557         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4558             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4559             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4560             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4561
4562             # for non online change, we try to hot-unplug
4563             die "skip\n" if !$hotplug;
4564             vm_deviceunplug($vmid, $conf, $opt);
4565         } else {
4566
4567             die "internal error" if $opt !~ m/net(\d+)/;
4568             my $iface = "tap${vmid}i$1";
4569
4570             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4571                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4572                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4573                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4574                 PVE::Network::tap_unplug($iface);
4575                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4576             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4577                 # Rate can be applied on its own but any change above needs to
4578                 # include the rate in tap_plug since OVS resets everything.
4579                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4580             }
4581
4582             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4583                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4584             }
4585
4586             return 1;
4587         }
4588     }
4589
4590     if ($hotplug) {
4591         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4592     } else {
4593         die "skip\n";
4594     }
4595 }
4596
4597 sub vmconfig_update_disk {
4598     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4599
4600     # fixme: do we need force?
4601
4602     my $drive = parse_drive($opt, $value);
4603
4604     if ($conf->{$opt}) {
4605
4606         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4607
4608             my $media = $drive->{media} || 'disk';
4609             my $oldmedia = $old_drive->{media} || 'disk';
4610             die "unable to change media type\n" if $media ne $oldmedia;
4611
4612             if (!drive_is_cdrom($old_drive)) {
4613
4614                 if ($drive->{file} ne $old_drive->{file}) {
4615
4616                     die "skip\n" if !$hotplug;
4617
4618                     # unplug and register as unused
4619                     vm_deviceunplug($vmid, $conf, $opt);
4620                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4621
4622                 } else {
4623                     # update existing disk
4624
4625                     # skip non hotpluggable value
4626                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4627                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4628                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4629                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4630                         die "skip\n";
4631                     }
4632
4633                     # apply throttle
4634                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4635                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4636                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4637                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4638                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4639                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4640                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4641                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4642                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4643                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4644                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4645                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4646                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4647                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4648                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4649                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4650                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4651                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4652
4653                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4654                                                    ($drive->{mbps} || 0)*1024*1024,
4655                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4656                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4657                                                    $drive->{iops} || 0,
4658                                                    $drive->{iops_rd} || 0,
4659                                                    $drive->{iops_wr} || 0,
4660                                                    ($drive->{mbps_max} || 0)*1024*1024,
4661                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4662                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4663                                                    $drive->{iops_max} || 0,
4664                                                    $drive->{iops_rd_max} || 0,
4665                                                    $drive->{iops_wr_max} || 0,
4666                                                    $drive->{bps_max_length} || 1,
4667                                                    $drive->{bps_rd_max_length} || 1,
4668                                                    $drive->{bps_wr_max_length} || 1,
4669                                                    $drive->{iops_max_length} || 1,
4670                                                    $drive->{iops_rd_max_length} || 1,
4671                                                    $drive->{iops_wr_max_length} || 1);
4672
4673                     }
4674
4675                     return 1;
4676                 }
4677
4678             } else { # cdrom
4679
4680                 if ($drive->{file} eq 'none') {
4681                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4682                 } else {
4683                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4684                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4685                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4686                 }
4687
4688                 return 1;
4689             }
4690         }
4691     }
4692
4693     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4694     # hotplug new disks
4695     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4696     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4697 }
4698
4699 sub vm_start {
4700     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4701         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4702
4703     PVE::QemuConfig->lock_config($vmid, sub {
4704         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4705
4706         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4707
4708         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4709
4710         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4711
4712         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4713             vmconfig_apply_pending($vmid, $conf, $storecfg);
4714             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4715         }
4716
4717         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4718
4719         my $defaults = load_defaults();
4720
4721         # set environment variable useful inside network script
4722         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4723
4724         my $local_volumes = {};
4725
4726         if ($targetstorage) {
4727             foreach_drive($conf, sub {
4728                 my ($ds, $drive) = @_;
4729
4730                 return if drive_is_cdrom($drive);
4731
4732                 my $volid = $drive->{file};
4733
4734                 return if !$volid;
4735
4736                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4737
4738                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4739                 return if $scfg->{shared};
4740                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4741             });
4742
4743             my $format = undef;
4744
4745             foreach my $opt (sort keys %$local_volumes) {
4746
4747                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4748                 my $drive = parse_drive($opt, $conf->{$opt});
4749
4750                 #if remote storage is specified, use default format
4751                 if ($targetstorage && $targetstorage ne "1") {
4752                     $storeid = $targetstorage;
4753                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4754                     $format = $defFormat;
4755                 } else {
4756                     #else we use same format than original
4757                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4758                     $format = qemu_img_format($scfg, $volid);
4759                 }
4760
4761                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4762                 my $newdrive = $drive;
4763                 $newdrive->{format} = $format;
4764                 $newdrive->{file} = $newvolid;
4765                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4766                 $local_volumes->{$opt} = $drivestr;
4767                 #pass drive to conf for command line
4768                 $conf->{$opt} = $drivestr;
4769             }
4770         }
4771
4772         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4773
4774         my $migrate_port = 0;
4775         my $migrate_uri;
4776         if ($statefile) {
4777             if ($statefile eq 'tcp') {
4778                 my $localip = "localhost";
4779                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4780                 my $nodename = PVE::INotify::nodename();
4781
4782                 if (!defined($migration_type)) {
4783                     if (defined($datacenterconf->{migration}->{type})) {
4784                         $migration_type = $datacenterconf->{migration}->{type};
4785                     } else {
4786                         $migration_type = 'secure';
4787                     }
4788                 }
4789
4790                 if ($migration_type eq 'insecure') {
4791                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4792                     if ($migrate_network_addr) {
4793                         $localip = $migrate_network_addr;
4794                     } else {
4795                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4796                     }
4797
4798                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4799                 }
4800
4801                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4802                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4803                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4804                 push @$cmd, '-incoming', $migrate_uri;
4805                 push @$cmd, '-S';
4806
4807             } elsif ($statefile eq 'unix') {
4808                 # should be default for secure migrations as a ssh TCP forward
4809                 # tunnel is not deterministic reliable ready and fails regurarly
4810                 # to set up in time, so use UNIX socket forwards
4811                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4812                 unlink $socket_addr;
4813
4814                 $migrate_uri = "unix:$socket_addr";
4815
4816                 push @$cmd, '-incoming', $migrate_uri;
4817                 push @$cmd, '-S';
4818
4819             } else {
4820                 push @$cmd, '-loadstate', $statefile;
4821             }
4822         } elsif ($paused) {
4823             push @$cmd, '-S';
4824         }
4825
4826         # host pci devices
4827         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4828           my $d = parse_hostpci($conf->{"hostpci$i"});
4829           next if !$d;
4830           my $pcidevices = $d->{pciid};
4831           foreach my $pcidevice (@$pcidevices) {
4832                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4833
4834                 my $info = pci_device_info("0000:$pciid");
4835                 die "IOMMU not present\n" if !check_iommu_support();
4836                 die "no pci device info for device '$pciid'\n" if !$info;
4837                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4838                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4839           }
4840         }
4841
4842         PVE::Storage::activate_volumes($storecfg, $vollist);
4843
4844         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4845             my $cmd = [];
4846             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4847             eval  { run_command($cmd); };
4848         }
4849
4850         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4851                                                   : $defaults->{cpuunits};
4852
4853         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4854         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4855
4856         my %properties = (
4857             Slice => 'qemu.slice',
4858             KillMode => 'none',
4859             CPUShares => $cpuunits
4860         );
4861
4862         if (my $cpulimit = $conf->{cpulimit}) {
4863             $properties{CPUQuota} = int($cpulimit * 100);
4864         }
4865         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4866
4867         if ($conf->{hugepages}) {
4868
4869             my $code = sub {
4870                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4871                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4872
4873                 PVE::QemuServer::Memory::hugepages_mount();
4874                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4875
4876                 eval  {
4877                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4878                     run_command($cmd, %run_params);
4879                 };
4880
4881                 if (my $err = $@) {
4882                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4883                     die $err;
4884                 }
4885
4886                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4887             };
4888             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4889
4890         } else {
4891             eval  {
4892                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4893                 run_command($cmd, %run_params);
4894             };
4895         }
4896
4897         if (my $err = $@) {
4898             # deactivate volumes if start fails
4899             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4900             die "start failed: $err";
4901         }
4902
4903         print "migration listens on $migrate_uri\n" if $migrate_uri;
4904
4905         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4906             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4907             warn $@ if $@;
4908         }
4909
4910         #start nbd server for storage migration
4911         if ($targetstorage) {
4912             my $nodename = PVE::INotify::nodename();
4913             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4914             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4915             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4916             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4917
4918             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4919
4920             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4921
4922             foreach my $opt (sort keys %$local_volumes) {
4923                 my $volid = $local_volumes->{$opt};
4924                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4925                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4926                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4927             }
4928         }
4929
4930         if ($migratedfrom) {
4931             eval {
4932                 set_migration_caps($vmid);
4933             };
4934             warn $@ if $@;
4935
4936             if ($spice_port) {
4937                 print "spice listens on port $spice_port\n";
4938                 if ($spice_ticket) {
4939                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4940                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4941                 }
4942             }
4943
4944         } else {
4945             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4946                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4947                     if $conf->{balloon};
4948             }
4949
4950             foreach my $opt (keys %$conf) {
4951                 next if $opt !~  m/^net\d+$/;
4952                 my $nicconf = parse_net($conf->{$opt});
4953                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4954             }
4955         }
4956
4957         vm_mon_cmd_nocheck($vmid, 'qom-set',
4958                     path => "machine/peripheral/balloon0",
4959                     property => "guest-stats-polling-interval",
4960                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4961
4962     });
4963 }
4964
4965 sub vm_mon_cmd {
4966     my ($vmid, $execute, %params) = @_;
4967
4968     my $cmd = { execute => $execute, arguments => \%params };
4969     vm_qmp_command($vmid, $cmd);
4970 }
4971
4972 sub vm_mon_cmd_nocheck {
4973     my ($vmid, $execute, %params) = @_;
4974
4975     my $cmd = { execute => $execute, arguments => \%params };
4976     vm_qmp_command($vmid, $cmd, 1);
4977 }
4978
4979 sub vm_qmp_command {
4980     my ($vmid, $cmd, $nocheck) = @_;
4981
4982     my $res;
4983
4984     my $timeout;
4985     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4986         $timeout = $cmd->{arguments}->{timeout};
4987         delete $cmd->{arguments}->{timeout};
4988     }
4989
4990     eval {
4991         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4992         my $sname = qmp_socket($vmid);
4993         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4994             my $qmpclient = PVE::QMPClient->new();
4995
4996             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4997         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4998             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4999                 if scalar(%{$cmd->{arguments}});
5000             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
5001         } else {
5002             die "unable to open monitor socket\n";
5003         }
5004     };
5005     if (my $err = $@) {
5006         syslog("err", "VM $vmid qmp command failed - $err");
5007         die $err;
5008     }
5009
5010     return $res;
5011 }
5012
5013 sub vm_human_monitor_command {
5014     my ($vmid, $cmdline) = @_;
5015
5016     my $res;
5017
5018     my $cmd = {
5019         execute => 'human-monitor-command',
5020         arguments => { 'command-line' => $cmdline},
5021     };
5022
5023     return vm_qmp_command($vmid, $cmd);
5024 }
5025
5026 sub vm_commandline {
5027     my ($storecfg, $vmid) = @_;
5028
5029     my $conf = PVE::QemuConfig->load_config($vmid);
5030
5031     my $defaults = load_defaults();
5032
5033     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5034
5035     return PVE::Tools::cmd2string($cmd);
5036 }
5037
5038 sub vm_reset {
5039     my ($vmid, $skiplock) = @_;
5040
5041     PVE::QemuConfig->lock_config($vmid, sub {
5042
5043         my $conf = PVE::QemuConfig->load_config($vmid);
5044
5045         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5046
5047         vm_mon_cmd($vmid, "system_reset");
5048     });
5049 }
5050
5051 sub get_vm_volumes {
5052     my ($conf) = @_;
5053
5054     my $vollist = [];
5055     foreach_volid($conf, sub {
5056         my ($volid, $attr) = @_;
5057
5058         return if $volid =~ m|^/|;
5059
5060         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5061         return if !$sid;
5062
5063         push @$vollist, $volid;
5064     });
5065
5066     return $vollist;
5067 }
5068
5069 sub vm_stop_cleanup {
5070     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5071
5072     eval {
5073
5074         if (!$keepActive) {
5075             my $vollist = get_vm_volumes($conf);
5076             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5077         }
5078
5079         foreach my $ext (qw(mon qmp pid vnc qga)) {
5080             unlink "/var/run/qemu-server/${vmid}.$ext";
5081         }
5082
5083         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5084     };
5085     warn $@ if $@; # avoid errors - just warn
5086 }
5087
5088 # Note: use $nockeck to skip tests if VM configuration file exists.
5089 # We need that when migration VMs to other nodes (files already moved)
5090 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5091 sub vm_stop {
5092     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5093
5094     $force = 1 if !defined($force) && !$shutdown;
5095
5096     if ($migratedfrom){
5097         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5098         kill 15, $pid if $pid;
5099         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5100         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5101         return;
5102     }
5103
5104     PVE::QemuConfig->lock_config($vmid, sub {
5105
5106         my $pid = check_running($vmid, $nocheck);
5107         return if !$pid;
5108
5109         my $conf;
5110         if (!$nocheck) {
5111             $conf = PVE::QemuConfig->load_config($vmid);
5112             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5113             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5114                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5115                 $timeout = $opts->{down} if $opts->{down};
5116             }
5117         }
5118
5119         $timeout = 60 if !defined($timeout);
5120
5121         eval {
5122             if ($shutdown) {
5123                 if (defined($conf) && $conf->{agent}) {
5124                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5125                 } else {
5126                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5127                 }
5128             } else {
5129                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5130             }
5131         };
5132         my $err = $@;
5133
5134         if (!$err) {
5135             my $count = 0;
5136             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5137                 $count++;
5138                 sleep 1;
5139             }
5140
5141             if ($count >= $timeout) {
5142                 if ($force) {
5143                     warn "VM still running - terminating now with SIGTERM\n";
5144                     kill 15, $pid;
5145                 } else {
5146                     die "VM quit/powerdown failed - got timeout\n";
5147                 }
5148             } else {
5149                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5150                 return;
5151             }
5152         } else {
5153             if ($force) {
5154                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5155                 kill 15, $pid;
5156             } else {
5157                 die "VM quit/powerdown failed\n";
5158             }
5159         }
5160
5161         # wait again
5162         $timeout = 10;
5163
5164         my $count = 0;
5165         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5166             $count++;
5167             sleep 1;
5168         }
5169
5170         if ($count >= $timeout) {
5171             warn "VM still running - terminating now with SIGKILL\n";
5172             kill 9, $pid;
5173             sleep 1;
5174         }
5175
5176         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5177    });
5178 }
5179
5180 sub vm_suspend {
5181     my ($vmid, $skiplock) = @_;
5182
5183     PVE::QemuConfig->lock_config($vmid, sub {
5184
5185         my $conf = PVE::QemuConfig->load_config($vmid);
5186
5187         PVE::QemuConfig->check_lock($conf)
5188             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5189
5190         vm_mon_cmd($vmid, "stop");
5191     });
5192 }
5193
5194 sub vm_resume {
5195     my ($vmid, $skiplock, $nocheck) = @_;
5196
5197     PVE::QemuConfig->lock_config($vmid, sub {
5198
5199         if (!$nocheck) {
5200
5201             my $conf = PVE::QemuConfig->load_config($vmid);
5202
5203             PVE::QemuConfig->check_lock($conf)
5204                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5205
5206             vm_mon_cmd($vmid, "cont");
5207
5208         } else {
5209             vm_mon_cmd_nocheck($vmid, "cont");
5210         }
5211     });
5212 }
5213
5214 sub vm_sendkey {
5215     my ($vmid, $skiplock, $key) = @_;
5216
5217     PVE::QemuConfig->lock_config($vmid, sub {
5218
5219         my $conf = PVE::QemuConfig->load_config($vmid);
5220
5221         # there is no qmp command, so we use the human monitor command
5222         vm_human_monitor_command($vmid, "sendkey $key");
5223     });
5224 }
5225
5226 sub vm_destroy {
5227     my ($storecfg, $vmid, $skiplock) = @_;
5228
5229     PVE::QemuConfig->lock_config($vmid, sub {
5230
5231         my $conf = PVE::QemuConfig->load_config($vmid);
5232
5233         if (!check_running($vmid)) {
5234             destroy_vm($storecfg, $vmid, undef, $skiplock);
5235         } else {
5236             die "VM $vmid is running - destroy failed\n";
5237         }
5238     });
5239 }
5240
5241 # pci helpers
5242
5243 sub file_write {
5244     my ($filename, $buf) = @_;
5245
5246     my $fh = IO::File->new($filename, "w");
5247     return undef if !$fh;
5248
5249     my $res = print $fh $buf;
5250
5251     $fh->close();
5252
5253     return $res;
5254 }
5255
5256 sub pci_device_info {
5257     my ($name) = @_;
5258
5259     my $res;
5260
5261     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5262     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5263
5264     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5265     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5266
5267     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5268     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5269
5270     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5271     return undef if !defined($product) || $product !~ s/^0x//;
5272
5273     $res = {
5274         name => $name,
5275         vendor => $vendor,
5276         product => $product,
5277         domain => $domain,
5278         bus => $bus,
5279         slot => $slot,
5280         func => $func,
5281         irq => $irq,
5282         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5283     };
5284
5285     return $res;
5286 }
5287
5288 sub pci_dev_reset {
5289     my ($dev) = @_;
5290
5291     my $name = $dev->{name};
5292
5293     my $fn = "$pcisysfs/devices/$name/reset";
5294
5295     return file_write($fn, "1");
5296 }
5297
5298 sub pci_dev_bind_to_vfio {
5299     my ($dev) = @_;
5300
5301     my $name = $dev->{name};
5302
5303     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5304
5305     if (!-d $vfio_basedir) {
5306         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5307     }
5308     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5309
5310     my $testdir = "$vfio_basedir/$name";
5311     return 1 if -d $testdir;
5312
5313     my $data = "$dev->{vendor} $dev->{product}";
5314     return undef if !file_write("$vfio_basedir/new_id", $data);
5315
5316     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5317     if (!file_write($fn, $name)) {
5318         return undef if -f $fn;
5319     }
5320
5321     $fn = "$vfio_basedir/bind";
5322     if (! -d $testdir) {
5323         return undef if !file_write($fn, $name);
5324     }
5325
5326     return -d $testdir;
5327 }
5328
5329 sub pci_dev_group_bind_to_vfio {
5330     my ($pciid) = @_;
5331
5332     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5333
5334     if (!-d $vfio_basedir) {
5335         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5336     }
5337     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5338
5339     # get IOMMU group devices
5340     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5341       my @devs = grep /^0000:/, readdir($D);
5342     closedir($D);
5343
5344     foreach my $pciid (@devs) {
5345         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5346
5347         # pci bridges, switches or root ports are not supported
5348         # they have a pci_bus subdirectory so skip them
5349         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5350
5351         my $info = pci_device_info($1);
5352         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5353     }
5354
5355     return 1;
5356 }
5357
5358 # vzdump restore implementaion
5359
5360 sub tar_archive_read_firstfile {
5361     my $archive = shift;
5362
5363     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5364
5365     # try to detect archive type first
5366     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5367         die "unable to open file '$archive'\n";
5368     my $firstfile = <$fh>;
5369     kill 15, $pid;
5370     close $fh;
5371
5372     die "ERROR: archive contaions no data\n" if !$firstfile;
5373     chomp $firstfile;
5374
5375     return $firstfile;
5376 }
5377
5378 sub tar_restore_cleanup {
5379     my ($storecfg, $statfile) = @_;
5380
5381     print STDERR "starting cleanup\n";
5382
5383     if (my $fd = IO::File->new($statfile, "r")) {
5384         while (defined(my $line = <$fd>)) {
5385             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5386                 my $volid = $2;
5387                 eval {
5388                     if ($volid =~ m|^/|) {
5389                         unlink $volid || die 'unlink failed\n';
5390                     } else {
5391                         PVE::Storage::vdisk_free($storecfg, $volid);
5392                     }
5393                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5394                 };
5395                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5396             } else {
5397                 print STDERR "unable to parse line in statfile - $line";
5398             }
5399         }
5400         $fd->close();
5401     }
5402 }
5403
5404 sub restore_archive {
5405     my ($archive, $vmid, $user, $opts) = @_;
5406
5407     my $format = $opts->{format};
5408     my $comp;
5409
5410     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5411         $format = 'tar' if !$format;
5412         $comp = 'gzip';
5413     } elsif ($archive =~ m/\.tar$/) {
5414         $format = 'tar' if !$format;
5415     } elsif ($archive =~ m/.tar.lzo$/) {
5416         $format = 'tar' if !$format;
5417         $comp = 'lzop';
5418     } elsif ($archive =~ m/\.vma$/) {
5419         $format = 'vma' if !$format;
5420     } elsif ($archive =~ m/\.vma\.gz$/) {
5421         $format = 'vma' if !$format;
5422         $comp = 'gzip';
5423     } elsif ($archive =~ m/\.vma\.lzo$/) {
5424         $format = 'vma' if !$format;
5425         $comp = 'lzop';
5426     } else {
5427         $format = 'vma' if !$format; # default
5428     }
5429
5430     # try to detect archive format
5431     if ($format eq 'tar') {
5432         return restore_tar_archive($archive, $vmid, $user, $opts);
5433     } else {
5434         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5435     }
5436 }
5437
5438 sub restore_update_config_line {
5439     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5440
5441     return if $line =~ m/^\#qmdump\#/;
5442     return if $line =~ m/^\#vzdump\#/;
5443     return if $line =~ m/^lock:/;
5444     return if $line =~ m/^unused\d+:/;
5445     return if $line =~ m/^parent:/;
5446     return if $line =~ m/^template:/; # restored VM is never a template
5447
5448     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5449     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5450         # try to convert old 1.X settings
5451         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5452         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5453             my ($model, $macaddr) = split(/\=/, $devconfig);
5454             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5455             my $net = {
5456                 model => $model,
5457                 bridge => "vmbr$ind",
5458                 macaddr => $macaddr,
5459             };
5460             my $netstr = print_net($net);
5461
5462             print $outfd "net$cookie->{netcount}: $netstr\n";
5463             $cookie->{netcount}++;
5464         }
5465     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5466         my ($id, $netstr) = ($1, $2);
5467         my $net = parse_net($netstr);
5468         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5469         $netstr = print_net($net);
5470         print $outfd "$id: $netstr\n";
5471     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5472         my $virtdev = $1;
5473         my $value = $3;
5474         my $di = parse_drive($virtdev, $value);
5475         if (defined($di->{backup}) && !$di->{backup}) {
5476             print $outfd "#$line";
5477         } elsif ($map->{$virtdev}) {
5478             delete $di->{format}; # format can change on restore
5479             $di->{file} = $map->{$virtdev};
5480             $value = print_drive($vmid, $di);
5481             print $outfd "$virtdev: $value\n";
5482         } else {
5483             print $outfd $line;
5484         }
5485     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5486         my ($uuid, $uuid_str);
5487         UUID::generate($uuid);
5488         UUID::unparse($uuid, $uuid_str);
5489         my $smbios1 = parse_smbios1($2);
5490         $smbios1->{uuid} = $uuid_str;
5491         print $outfd $1.print_smbios1($smbios1)."\n";
5492     } else {
5493         print $outfd $line;
5494     }
5495 }
5496
5497 sub scan_volids {
5498     my ($cfg, $vmid) = @_;
5499
5500     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5501
5502     my $volid_hash = {};
5503     foreach my $storeid (keys %$info) {
5504         foreach my $item (@{$info->{$storeid}}) {
5505             next if !($item->{volid} && $item->{size});
5506             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5507             $volid_hash->{$item->{volid}} = $item;
5508         }
5509     }
5510
5511     return $volid_hash;
5512 }
5513
5514 sub is_volume_in_use {
5515     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5516
5517     my $path = PVE::Storage::path($storecfg, $volid);
5518
5519     my $scan_config = sub {
5520         my ($cref, $snapname) = @_;
5521
5522         foreach my $key (keys %$cref) {
5523             my $value = $cref->{$key};
5524             if (is_valid_drivename($key)) {
5525                 next if $skip_drive && $key eq $skip_drive;
5526                 my $drive = parse_drive($key, $value);
5527                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5528                 return 1 if $volid eq $drive->{file};
5529                 if ($drive->{file} =~ m!^/!) {
5530                     return 1 if $drive->{file} eq $path;
5531                 } else {
5532                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5533                     next if !$storeid;
5534                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5535                     next if !$scfg;
5536                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5537                 }
5538             }
5539         }
5540
5541         return 0;
5542     };
5543
5544     return 1 if &$scan_config($conf);
5545
5546     undef $skip_drive;
5547
5548     foreach my $snapname (keys %{$conf->{snapshots}}) {
5549         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5550     }
5551
5552     return 0;
5553 }
5554
5555 sub update_disksize {
5556     my ($vmid, $conf, $volid_hash) = @_;
5557
5558     my $changes;
5559
5560     # used and unused disks
5561     my $referenced = {};
5562
5563     # Note: it is allowed to define multiple storages with same path (alias), so
5564     # we need to check both 'volid' and real 'path' (two different volid can point
5565     # to the same path).
5566
5567     my $referencedpath = {};
5568
5569     # update size info
5570     foreach my $opt (keys %$conf) {
5571         if (is_valid_drivename($opt)) {
5572             my $drive = parse_drive($opt, $conf->{$opt});
5573             my $volid = $drive->{file};
5574             next if !$volid;
5575
5576             $referenced->{$volid} = 1;
5577             if ($volid_hash->{$volid} &&
5578                 (my $path = $volid_hash->{$volid}->{path})) {
5579                 $referencedpath->{$path} = 1;
5580             }
5581
5582             next if drive_is_cdrom($drive);
5583             next if !$volid_hash->{$volid};
5584
5585             $drive->{size} = $volid_hash->{$volid}->{size};
5586             my $new = print_drive($vmid, $drive);
5587             if ($new ne $conf->{$opt}) {
5588                 $changes = 1;
5589                 $conf->{$opt} = $new;
5590             }
5591         }
5592     }
5593
5594     # remove 'unusedX' entry if volume is used
5595     foreach my $opt (keys %$conf) {
5596         next if $opt !~ m/^unused\d+$/;
5597         my $volid = $conf->{$opt};
5598         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5599         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5600             $changes = 1;
5601             delete $conf->{$opt};
5602         }
5603
5604         $referenced->{$volid} = 1;
5605         $referencedpath->{$path} = 1 if $path;
5606     }
5607
5608     foreach my $volid (sort keys %$volid_hash) {
5609         next if $volid =~ m/vm-$vmid-state-/;
5610         next if $referenced->{$volid};
5611         my $path = $volid_hash->{$volid}->{path};
5612         next if !$path; # just to be sure
5613         next if $referencedpath->{$path};
5614         $changes = 1;
5615         PVE::QemuConfig->add_unused_volume($conf, $volid);
5616         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5617     }
5618
5619     return $changes;
5620 }
5621
5622 sub rescan {
5623     my ($vmid, $nolock) = @_;
5624
5625     my $cfg = PVE::Storage::config();
5626
5627     my $volid_hash = scan_volids($cfg, $vmid);
5628
5629     my $updatefn =  sub {
5630         my ($vmid) = @_;
5631
5632         my $conf = PVE::QemuConfig->load_config($vmid);
5633
5634         PVE::QemuConfig->check_lock($conf);
5635
5636         my $vm_volids = {};
5637         foreach my $volid (keys %$volid_hash) {
5638             my $info = $volid_hash->{$volid};
5639             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5640         }
5641
5642         my $changes = update_disksize($vmid, $conf, $vm_volids);
5643
5644         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5645     };
5646
5647     if (defined($vmid)) {
5648         if ($nolock) {
5649             &$updatefn($vmid);
5650         } else {
5651             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5652         }
5653     } else {
5654         my $vmlist = config_list();
5655         foreach my $vmid (keys %$vmlist) {
5656             if ($nolock) {
5657                 &$updatefn($vmid);
5658             } else {
5659                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5660             }
5661         }
5662     }
5663 }
5664
5665 sub restore_vma_archive {
5666     my ($archive, $vmid, $user, $opts, $comp) = @_;
5667
5668     my $input = $archive eq '-' ? "<&STDIN" : undef;
5669     my $readfrom = $archive;
5670
5671     my $uncomp = '';
5672     if ($comp) {
5673         $readfrom = '-';
5674         my $qarchive = PVE::Tools::shellquote($archive);
5675         if ($comp eq 'gzip') {
5676             $uncomp = "zcat $qarchive|";
5677         } elsif ($comp eq 'lzop') {
5678             $uncomp = "lzop -d -c $qarchive|";
5679         } else {
5680             die "unknown compression method '$comp'\n";
5681         }
5682
5683     }
5684
5685     my $tmpdir = "/var/tmp/vzdumptmp$$";
5686     rmtree $tmpdir;
5687
5688     # disable interrupts (always do cleanups)
5689     local $SIG{INT} =
5690         local $SIG{TERM} =
5691         local $SIG{QUIT} =
5692         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5693
5694     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5695     POSIX::mkfifo($mapfifo, 0600);
5696     my $fifofh;
5697
5698     my $openfifo = sub {
5699         open($fifofh, '>', $mapfifo) || die $!;
5700     };
5701
5702     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5703
5704     my $oldtimeout;
5705     my $timeout = 5;
5706
5707     my $devinfo = {};
5708
5709     my $rpcenv = PVE::RPCEnvironment::get();
5710
5711     my $conffile = PVE::QemuConfig->config_file($vmid);
5712     my $tmpfn = "$conffile.$$.tmp";
5713
5714     # Note: $oldconf is undef if VM does not exists
5715     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5716     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5717
5718     my $print_devmap = sub {
5719         my $virtdev_hash = {};
5720
5721         my $cfgfn = "$tmpdir/qemu-server.conf";
5722
5723         # we can read the config - that is already extracted
5724         my $fh = IO::File->new($cfgfn, "r") ||
5725             "unable to read qemu-server.conf - $!\n";
5726
5727         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5728         if (-f $fwcfgfn) {
5729             my $pve_firewall_dir = '/etc/pve/firewall';
5730             mkdir $pve_firewall_dir; # make sure the dir exists
5731             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5732         }
5733
5734         while (defined(my $line = <$fh>)) {
5735             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5736                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5737                 die "archive does not contain data for drive '$virtdev'\n"
5738                     if !$devinfo->{$devname};
5739                 if (defined($opts->{storage})) {
5740                     $storeid = $opts->{storage} || 'local';
5741                 } elsif (!$storeid) {
5742                     $storeid = 'local';
5743                 }
5744                 $format = 'raw' if !$format;
5745                 $devinfo->{$devname}->{devname} = $devname;
5746                 $devinfo->{$devname}->{virtdev} = $virtdev;
5747                 $devinfo->{$devname}->{format} = $format;
5748                 $devinfo->{$devname}->{storeid} = $storeid;
5749
5750                 # check permission on storage
5751                 my $pool = $opts->{pool}; # todo: do we need that?
5752                 if ($user ne 'root@pam') {
5753                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5754                 }
5755
5756                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5757             }
5758         }
5759
5760         foreach my $devname (keys %$devinfo) {
5761             die "found no device mapping information for device '$devname'\n"
5762                 if !$devinfo->{$devname}->{virtdev};
5763         }
5764
5765         my $cfg = PVE::Storage::config();
5766
5767         # create empty/temp config
5768         if ($oldconf) {
5769             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5770             foreach_drive($oldconf, sub {
5771                 my ($ds, $drive) = @_;
5772
5773                 return if drive_is_cdrom($drive);
5774
5775                 my $volid = $drive->{file};
5776
5777                 return if !$volid || $volid =~ m|^/|;
5778
5779                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5780                 return if !$path || !$owner || ($owner != $vmid);
5781
5782                 # Note: only delete disk we want to restore
5783                 # other volumes will become unused
5784                 if ($virtdev_hash->{$ds}) {
5785                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5786                     if (my $err = $@) {
5787                         warn $err;
5788                     }
5789                 }
5790             });
5791
5792             # delete vmstate files
5793             # since after the restore we have no snapshots anymore
5794             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5795                 my $snap = $oldconf->{snapshots}->{$snapname};
5796                 if ($snap->{vmstate}) {
5797                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5798                     if (my $err = $@) {
5799                         warn $err;
5800                     }
5801                 }
5802             }
5803         }
5804
5805         my $map = {};
5806         foreach my $virtdev (sort keys %$virtdev_hash) {
5807             my $d = $virtdev_hash->{$virtdev};
5808             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5809             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5810
5811             # test if requested format is supported
5812             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5813             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5814             $d->{format} = $defFormat if !$supported;
5815
5816             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5817                                                   $d->{format}, undef, $alloc_size);
5818             print STDERR "new volume ID is '$volid'\n";
5819             $d->{volid} = $volid;
5820             my $path = PVE::Storage::path($cfg, $volid);
5821
5822             PVE::Storage::activate_volumes($cfg,[$volid]);
5823
5824             my $write_zeros = 1;
5825             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5826                 $write_zeros = 0;
5827             }
5828
5829             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5830
5831             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5832             $map->{$virtdev} = $volid;
5833         }
5834
5835         $fh->seek(0, 0) || die "seek failed - $!\n";
5836
5837         my $outfd = new IO::File ($tmpfn, "w") ||
5838             die "unable to write config for VM $vmid\n";
5839
5840         my $cookie = { netcount => 0 };
5841         while (defined(my $line = <$fh>)) {
5842             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5843         }
5844
5845         $fh->close();
5846         $outfd->close();
5847     };
5848
5849     eval {
5850         # enable interrupts
5851         local $SIG{INT} =
5852             local $SIG{TERM} =
5853             local $SIG{QUIT} =
5854             local $SIG{HUP} =
5855             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5856         local $SIG{ALRM} = sub { die "got timeout\n"; };
5857
5858         $oldtimeout = alarm($timeout);
5859
5860         my $parser = sub {
5861             my $line = shift;
5862
5863             print "$line\n";
5864
5865             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5866                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5867                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5868             } elsif ($line =~ m/^CTIME: /) {
5869                 # we correctly received the vma config, so we can disable
5870                 # the timeout now for disk allocation (set to 10 minutes, so
5871                 # that we always timeout if something goes wrong)
5872                 alarm(600);
5873                 &$print_devmap();
5874                 print $fifofh "done\n";
5875                 my $tmp = $oldtimeout || 0;
5876                 $oldtimeout = undef;
5877                 alarm($tmp);
5878                 close($fifofh);
5879             }
5880         };
5881
5882         print "restore vma archive: $cmd\n";
5883         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5884     };
5885     my $err = $@;
5886
5887     alarm($oldtimeout) if $oldtimeout;
5888
5889     my $vollist = [];
5890     foreach my $devname (keys %$devinfo) {
5891         my $volid = $devinfo->{$devname}->{volid};
5892         push @$vollist, $volid if $volid;
5893     }
5894
5895     my $cfg = PVE::Storage::config();
5896     PVE::Storage::deactivate_volumes($cfg, $vollist);
5897
5898     unlink $mapfifo;
5899
5900     if ($err) {
5901         rmtree $tmpdir;
5902         unlink $tmpfn;
5903
5904         foreach my $devname (keys %$devinfo) {
5905             my $volid = $devinfo->{$devname}->{volid};
5906             next if !$volid;
5907             eval {
5908                 if ($volid =~ m|^/|) {
5909                     unlink $volid || die 'unlink failed\n';
5910                 } else {
5911                     PVE::Storage::vdisk_free($cfg, $volid);
5912                 }
5913                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5914             };
5915             print STDERR "unable to cleanup '$volid' - $@" if $@;
5916         }
5917         die $err;
5918     }
5919
5920     rmtree $tmpdir;
5921
5922     rename($tmpfn, $conffile) ||
5923         die "unable to commit configuration file '$conffile'\n";
5924
5925     PVE::Cluster::cfs_update(); # make sure we read new file
5926
5927     eval { rescan($vmid, 1); };
5928     warn $@ if $@;
5929 }
5930
5931 sub restore_tar_archive {
5932     my ($archive, $vmid, $user, $opts) = @_;
5933
5934     if ($archive ne '-') {
5935         my $firstfile = tar_archive_read_firstfile($archive);
5936         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5937             if $firstfile ne 'qemu-server.conf';
5938     }
5939
5940     my $storecfg = PVE::Storage::config();
5941
5942     # destroy existing data - keep empty config
5943     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5944     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5945
5946     my $tocmd = "/usr/lib/qemu-server/qmextract";
5947
5948     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5949     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5950     $tocmd .= ' --prealloc' if $opts->{prealloc};
5951     $tocmd .= ' --info' if $opts->{info};
5952
5953     # tar option "xf" does not autodetect compression when read from STDIN,
5954     # so we pipe to zcat
5955     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5956         PVE::Tools::shellquote("--to-command=$tocmd");
5957
5958     my $tmpdir = "/var/tmp/vzdumptmp$$";
5959     mkpath $tmpdir;
5960
5961     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5962     local $ENV{VZDUMP_VMID} = $vmid;
5963     local $ENV{VZDUMP_USER} = $user;
5964
5965     my $conffile = PVE::QemuConfig->config_file($vmid);
5966     my $tmpfn = "$conffile.$$.tmp";
5967
5968     # disable interrupts (always do cleanups)
5969     local $SIG{INT} =
5970         local $SIG{TERM} =
5971         local $SIG{QUIT} =
5972         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5973
5974     eval {
5975         # enable interrupts
5976         local $SIG{INT} =
5977             local $SIG{TERM} =
5978             local $SIG{QUIT} =
5979             local $SIG{HUP} =
5980             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5981
5982         if ($archive eq '-') {
5983             print "extracting archive from STDIN\n";
5984             run_command($cmd, input => "<&STDIN");
5985         } else {
5986             print "extracting archive '$archive'\n";
5987             run_command($cmd);
5988         }
5989
5990         return if $opts->{info};
5991
5992         # read new mapping
5993         my $map = {};
5994         my $statfile = "$tmpdir/qmrestore.stat";
5995         if (my $fd = IO::File->new($statfile, "r")) {
5996             while (defined (my $line = <$fd>)) {
5997                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5998                     $map->{$1} = $2 if $1;
5999                 } else {
6000                     print STDERR "unable to parse line in statfile - $line\n";
6001                 }
6002             }
6003             $fd->close();
6004         }
6005
6006         my $confsrc = "$tmpdir/qemu-server.conf";
6007
6008         my $srcfd = new IO::File($confsrc, "r") ||
6009             die "unable to open file '$confsrc'\n";
6010
6011         my $outfd = new IO::File ($tmpfn, "w") ||
6012             die "unable to write config for VM $vmid\n";
6013
6014         my $cookie = { netcount => 0 };
6015         while (defined (my $line = <$srcfd>)) {
6016             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6017         }
6018
6019         $srcfd->close();
6020         $outfd->close();
6021     };
6022     my $err = $@;
6023
6024     if ($err) {
6025
6026         unlink $tmpfn;
6027
6028         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6029
6030         die $err;
6031     }
6032
6033     rmtree $tmpdir;
6034
6035     rename $tmpfn, $conffile ||
6036         die "unable to commit configuration file '$conffile'\n";
6037
6038     PVE::Cluster::cfs_update(); # make sure we read new file
6039
6040     eval { rescan($vmid, 1); };
6041     warn $@ if $@;
6042 };
6043
6044 sub foreach_storage_used_by_vm {
6045     my ($conf, $func) = @_;
6046
6047     my $sidhash = {};
6048
6049     foreach_drive($conf, sub {
6050         my ($ds, $drive) = @_;
6051         return if drive_is_cdrom($drive);
6052
6053         my $volid = $drive->{file};
6054
6055         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6056         $sidhash->{$sid} = $sid if $sid;
6057     });
6058
6059     foreach my $sid (sort keys %$sidhash) {
6060         &$func($sid);
6061     }
6062 }
6063
6064 sub do_snapshots_with_qemu {
6065     my ($storecfg, $volid) = @_;
6066
6067     my $storage_name = PVE::Storage::parse_volume_id($volid);
6068
6069     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6070         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6071         return 1;
6072     }
6073
6074     if ($volid =~ m/\.(qcow2|qed)$/){
6075         return 1;
6076     }
6077
6078     return undef;
6079 }
6080
6081 sub qga_check_running {
6082     my ($vmid) = @_;
6083
6084     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6085     if ($@) {
6086         warn "Qemu Guest Agent is not running - $@";
6087         return 0;
6088     }
6089     return 1;
6090 }
6091
6092 sub template_create {
6093     my ($vmid, $conf, $disk) = @_;
6094
6095     my $storecfg = PVE::Storage::config();
6096
6097     foreach_drive($conf, sub {
6098         my ($ds, $drive) = @_;
6099
6100         return if drive_is_cdrom($drive);
6101         return if $disk && $ds ne $disk;
6102
6103         my $volid = $drive->{file};
6104         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6105
6106         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6107         $drive->{file} = $voliddst;
6108         $conf->{$ds} = print_drive($vmid, $drive);
6109         PVE::QemuConfig->write_config($vmid, $conf);
6110     });
6111 }
6112
6113 sub qemu_img_convert {
6114     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6115
6116     my $storecfg = PVE::Storage::config();
6117     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6118     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6119
6120     if ($src_storeid && $dst_storeid) {
6121
6122         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6123
6124         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6125         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6126
6127         my $src_format = qemu_img_format($src_scfg, $src_volname);
6128         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6129
6130         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6131         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6132
6133         my $cmd = [];
6134         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6135         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6136         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6137         if ($is_zero_initialized) {
6138             push @$cmd, "zeroinit:$dst_path";
6139         } else {
6140             push @$cmd, $dst_path;
6141         }
6142
6143         my $parser = sub {
6144             my $line = shift;
6145             if($line =~ m/\((\S+)\/100\%\)/){
6146                 my $percent = $1;
6147                 my $transferred = int($size * $percent / 100);
6148                 my $remaining = $size - $transferred;
6149
6150                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6151             }
6152
6153         };
6154
6155         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6156         my $err = $@;
6157         die "copy failed: $err" if $err;
6158     }
6159 }
6160
6161 sub qemu_img_format {
6162     my ($scfg, $volname) = @_;
6163
6164     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6165         return $1;
6166     } else {
6167         return "raw";
6168     }
6169 }
6170
6171 sub qemu_drive_mirror {
6172     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6173
6174     $jobs = {} if !$jobs;
6175
6176     my $qemu_target;
6177     my $format;
6178     $jobs->{"drive-$drive"} = {};
6179
6180     if ($dst_volid =~ /^nbd:/) {
6181         $qemu_target = $dst_volid;
6182         $format = "nbd";
6183     } else {
6184         my $storecfg = PVE::Storage::config();
6185         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6186
6187         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6188
6189         $format = qemu_img_format($dst_scfg, $dst_volname);
6190
6191         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6192
6193         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6194     }
6195
6196     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6197     $opts->{format} = $format if $format;
6198
6199     print "drive mirror is starting for drive-$drive\n";
6200
6201     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6202
6203     if (my $err = $@) {
6204         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6205         die "mirroring error: $err";
6206     }
6207
6208     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6209 }
6210
6211 sub qemu_drive_mirror_monitor {
6212     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6213
6214     eval {
6215         my $err_complete = 0;
6216
6217         while (1) {
6218             die "storage migration timed out\n" if $err_complete > 300;
6219
6220             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6221
6222             my $running_mirror_jobs = {};
6223             foreach my $stat (@$stats) {
6224                 next if $stat->{type} ne 'mirror';
6225                 $running_mirror_jobs->{$stat->{device}} = $stat;
6226             }
6227
6228             my $readycounter = 0;
6229
6230             foreach my $job (keys %$jobs) {
6231
6232                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6233                     print "$job : finished\n";
6234                     delete $jobs->{$job};
6235                     next;
6236                 }
6237
6238                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6239
6240                 my $busy = $running_mirror_jobs->{$job}->{busy};
6241                 my $ready = $running_mirror_jobs->{$job}->{ready};
6242                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6243                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6244                     my $remaining = $total - $transferred;
6245                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6246
6247                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6248                 }
6249
6250                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6251             }
6252
6253             last if scalar(keys %$jobs) == 0;
6254
6255             if ($readycounter == scalar(keys %$jobs)) {
6256                 print "all mirroring jobs are ready \n";
6257                 last if $skipcomplete; #do the complete later
6258
6259                 if ($vmiddst && $vmiddst != $vmid) {
6260                     my $agent_running = $qga && qga_check_running($vmid);
6261                     if ($agent_running) {
6262                         print "freeze filesystem\n";
6263                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6264                     } else {
6265                         print "suspend vm\n";
6266                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6267                     }
6268
6269                     # if we clone a disk for a new target vm, we don't switch the disk
6270                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6271
6272                     if ($agent_running) {
6273                         print "unfreeze filesystem\n";
6274                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6275                     } else {
6276                         print "resume vm\n";
6277                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6278                     }
6279
6280                     last;
6281                 } else {
6282
6283                     foreach my $job (keys %$jobs) {
6284                         # try to switch the disk if source and destination are on the same guest
6285                         print "$job: Completing block job...\n";
6286
6287                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6288                         if ($@ =~ m/cannot be completed/) {
6289                             print "$job: Block job cannot be completed, try again.\n";
6290                             $err_complete++;
6291                         }else {
6292                             print "$job: Completed successfully.\n";
6293                             $jobs->{$job}->{complete} = 1;
6294                         }
6295                     }
6296                 }
6297             }
6298             sleep 1;
6299         }
6300     };
6301     my $err = $@;
6302
6303     if ($err) {
6304         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6305         die "mirroring error: $err";
6306     }
6307
6308 }
6309
6310 sub qemu_blockjobs_cancel {
6311     my ($vmid, $jobs) = @_;
6312
6313     foreach my $job (keys %$jobs) {
6314         print "$job: Cancelling block job\n";
6315         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6316         $jobs->{$job}->{cancel} = 1;
6317     }
6318
6319     while (1) {
6320         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6321
6322         my $running_jobs = {};
6323         foreach my $stat (@$stats) {
6324             $running_jobs->{$stat->{device}} = $stat;
6325         }
6326
6327         foreach my $job (keys %$jobs) {
6328
6329             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6330                 print "$job: Done.\n";
6331                 delete $jobs->{$job};
6332             }
6333         }
6334
6335         last if scalar(keys %$jobs) == 0;
6336
6337         sleep 1;
6338     }
6339 }
6340
6341 sub clone_disk {
6342     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6343         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6344
6345     my $newvolid;
6346
6347     if (!$full) {
6348         print "create linked clone of drive $drivename ($drive->{file})\n";
6349         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6350         push @$newvollist, $newvolid;
6351     } else {
6352
6353         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6354         $storeid = $storage if $storage;
6355
6356         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6357         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6358
6359         print "create full clone of drive $drivename ($drive->{file})\n";
6360         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6361         push @$newvollist, $newvolid;
6362
6363         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6364
6365         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6366         if (!$running || $snapname) {
6367             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6368         } else {
6369
6370             my $kvmver = get_running_qemu_version ($vmid);
6371             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6372                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6373                     if $drive->{iothread};
6374             }
6375
6376             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6377         }
6378     }
6379
6380     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6381
6382     my $disk = $drive;
6383     $disk->{format} = undef;
6384     $disk->{file} = $newvolid;
6385     $disk->{size} = $size;
6386
6387     return $disk;
6388 }
6389
6390 # this only works if VM is running
6391 sub get_current_qemu_machine {
6392     my ($vmid) = @_;
6393
6394     my $cmd = { execute => 'query-machines', arguments => {} };
6395     my $res = vm_qmp_command($vmid, $cmd);
6396
6397     my ($current, $default);
6398     foreach my $e (@$res) {
6399         $default = $e->{name} if $e->{'is-default'};
6400         $current = $e->{name} if $e->{'is-current'};
6401     }
6402
6403     # fallback to the default machine if current is not supported by qemu
6404     return $current || $default || 'pc';
6405 }
6406
6407 sub get_running_qemu_version {
6408     my ($vmid) = @_;
6409     my $cmd = { execute => 'query-version', arguments => {} };
6410     my $res = vm_qmp_command($vmid, $cmd);
6411     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6412 }
6413
6414 sub qemu_machine_feature_enabled {
6415     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6416
6417     my $current_major;
6418     my $current_minor;
6419
6420     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6421
6422         $current_major = $3;
6423         $current_minor = $4;
6424
6425     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6426
6427         $current_major = $1;
6428         $current_minor = $2;
6429     }
6430
6431     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6432
6433
6434 }
6435
6436 sub qemu_machine_pxe {
6437     my ($vmid, $conf, $machine) = @_;
6438
6439     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6440
6441     foreach my $opt (keys %$conf) {
6442         next if $opt !~ m/^net(\d+)$/;
6443         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6444         next if !$net;
6445         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6446         return $machine.".pxe" if $romfile =~ m/pxe/;
6447         last;
6448     }
6449
6450     return $machine;
6451 }
6452
6453 sub qemu_use_old_bios_files {
6454     my ($machine_type) = @_;
6455
6456     return if !$machine_type;
6457
6458     my $use_old_bios_files = undef;
6459
6460     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6461         $machine_type = $1;
6462         $use_old_bios_files = 1;
6463     } else {
6464         my $kvmver = kvm_user_version();
6465         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6466         # load new efi bios files on migration. So this hack is required to allow
6467         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6468         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6469         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6470     }
6471
6472     return ($use_old_bios_files, $machine_type);
6473 }
6474
6475 sub create_efidisk {
6476     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6477
6478     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6479
6480     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6481     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6482     PVE::Storage::activate_volumes($storecfg, [$volid]);
6483
6484     my $path = PVE::Storage::path($storecfg, $volid);
6485     eval {
6486         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6487     };
6488     die "Copying EFI vars image failed: $@" if $@;
6489
6490     return ($volid, $vars_size);
6491 }
6492
6493 sub lspci {
6494
6495     my $devices = {};
6496
6497     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6498             my (undef, $id, $function) = @_;
6499             my $res = { id => $id, function => $function};
6500             push @{$devices->{$id}}, $res;
6501     });
6502
6503     # Entries should be sorted by functions.
6504     foreach my $id (keys %$devices) {
6505         my $dev = $devices->{$id};
6506         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6507     }
6508
6509     return $devices;
6510 }
6511
6512 sub vm_iothreads_list {
6513     my ($vmid) = @_;
6514
6515     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6516
6517     my $iothreads = {};
6518     foreach my $iothread (@$res) {
6519         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6520     }
6521
6522     return $iothreads;
6523 }
6524
6525 sub scsihw_infos {
6526     my ($conf, $drive) = @_;
6527
6528     my $maxdev = 0;
6529
6530     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6531         $maxdev = 7;
6532     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6533         $maxdev = 1;
6534     } else {
6535         $maxdev = 256;
6536     }
6537
6538     my $controller = int($drive->{index} / $maxdev);
6539     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6540
6541     return ($maxdev, $controller, $controller_prefix);
6542 }
6543
6544 sub add_hyperv_enlightenments {
6545     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6546
6547     return if $winversion < 6;
6548     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6549
6550     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6551
6552     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6553         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6554         push @$cpuFlags , 'hv_vapic';
6555         push @$cpuFlags , 'hv_time';
6556     } else {
6557         push @$cpuFlags , 'hv_spinlocks=0xffff';
6558     }
6559
6560     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6561         push @$cpuFlags , 'hv_reset';
6562         push @$cpuFlags , 'hv_vpindex';
6563         push @$cpuFlags , 'hv_runtime';
6564     }
6565
6566     if ($winversion >= 7) {
6567         push @$cpuFlags , 'hv_relaxed';
6568     }
6569 }
6570
6571 sub windows_version {
6572     my ($ostype) = @_;
6573
6574     return 0 if !$ostype;
6575
6576     my $winversion = 0;
6577
6578     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6579         $winversion = 5;
6580     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6581         $winversion = 6;
6582     } elsif ($ostype =~ m/^win(\d+)$/) {
6583         $winversion = $1;
6584     }
6585
6586     return $winversion;
6587 }
6588
6589 sub resolve_dst_disk_format {
6590         my ($storecfg, $storeid, $src_volname, $format) = @_;
6591         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6592
6593         if (!$format) {
6594             # if no target format is specified, use the source disk format as hint
6595             if ($src_volname) {
6596                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6597                 $format = qemu_img_format($scfg, $src_volname);
6598             } else {
6599                 return $defFormat;
6600             }
6601         }
6602
6603         # test if requested format is supported - else use default
6604         my $supported = grep { $_ eq $format } @$validFormats;
6605         $format = $defFormat if !$supported;
6606         return $format;
6607 }
6608
6609 sub resolve_first_disk {
6610     my $conf = shift;
6611     my @disks = PVE::QemuServer::valid_drive_names();
6612     my $firstdisk;
6613     foreach my $ds (reverse @disks) {
6614         next if !$conf->{$ds};
6615         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6616         next if PVE::QemuServer::drive_is_cdrom($disk);
6617         $firstdisk = $ds;
6618     }
6619     return $firstdisk;
6620 }
6621
6622 sub generate_smbios1_uuid {
6623     my ($uuid, $uuid_str);
6624     UUID::generate($uuid);
6625     UUID::unparse($uuid, $uuid_str);
6626     return "uuid=$uuid_str";
6627 }
6628
6629 # bash completion helper
6630
6631 sub complete_backup_archives {
6632     my ($cmdname, $pname, $cvalue) = @_;
6633
6634     my $cfg = PVE::Storage::config();
6635
6636     my $storeid;
6637
6638     if ($cvalue =~ m/^([^:]+):/) {
6639         $storeid = $1;
6640     }
6641
6642     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6643
6644     my $res = [];
6645     foreach my $id (keys %$data) {
6646         foreach my $item (@{$data->{$id}}) {
6647             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6648             push @$res, $item->{volid} if defined($item->{volid});
6649         }
6650     }
6651
6652     return $res;
6653 }
6654
6655 my $complete_vmid_full = sub {
6656     my ($running) = @_;
6657
6658     my $idlist = vmstatus();
6659
6660     my $res = [];
6661
6662     foreach my $id (keys %$idlist) {
6663         my $d = $idlist->{$id};
6664         if (defined($running)) {
6665             next if $d->{template};
6666             next if $running && $d->{status} ne 'running';
6667             next if !$running && $d->{status} eq 'running';
6668         }
6669         push @$res, $id;
6670
6671     }
6672     return $res;
6673 };
6674
6675 sub complete_vmid {
6676     return &$complete_vmid_full();
6677 }
6678
6679 sub complete_vmid_stopped {
6680     return &$complete_vmid_full(0);
6681 }
6682
6683 sub complete_vmid_running {
6684     return &$complete_vmid_full(1);
6685 }
6686
6687 sub complete_storage {
6688
6689     my $cfg = PVE::Storage::config();
6690     my $ids = $cfg->{ids};
6691
6692     my $res = [];
6693     foreach my $sid (keys %$ids) {
6694         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6695         next if !$ids->{$sid}->{content}->{images};
6696         push @$res, $sid;
6697     }
6698
6699     return $res;
6700 }
6701
6702 1;