PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     searchdomain => {
 544         optional => 1,
 545         type => 'string',
 546         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 547     },
 548     nameserver => {
 549         optional => 1,
 550         type => 'string', format => 'address-list',
 551         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 552     },
 553     sshkeys => {
 554         optional => 1,
 555         type => 'string',
 556         format => 'urlencoded',
 557         description => "cloud-init : Setup public SSH keys (one key per line, " .
 558                         "OpenSSH format).",
 559     },
 560     hostname => {
 561         optional => 1,
 562         description => "cloud-init: Hostname to use instead of the vm-name + search-domain.",
 563         type => 'string', format => 'dns-name',
 564         maxLength => 255,
 565     },
 566 };
 567
 568 # what about other qemu settings ?
 569 #cpu => 'string',
 570 #machine => 'string',
 571 #fda => 'file',
 572 #fdb => 'file',
 573 #mtdblock => 'file',
 574 #sd => 'file',
 575 #pflash => 'file',
 576 #snapshot => 'bool',
 577 #bootp => 'file',
 578 ##tftp => 'dir',
 579 ##smb => 'dir',
 580 #kernel => 'file',
 581 #append => 'string',
 582 #initrd => 'file',
 583 ##soundhw => 'string',
 584
 585 while (my ($k, $v) = each %$confdesc) {
 586     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 587 }
 588
 589 my $MAX_IDE_DISKS = 4;
 590 my $MAX_SCSI_DISKS = 14;
 591 my $MAX_VIRTIO_DISKS = 16;
 592 my $MAX_SATA_DISKS = 6;
 593 my $MAX_USB_DEVICES = 5;
 594 my $MAX_NETS = 32;
 595 my $MAX_UNUSED_DISKS = 8;
 596 my $MAX_HOSTPCI_DEVICES = 4;
 597 my $MAX_SERIAL_PORTS = 4;
 598 my $MAX_PARALLEL_PORTS = 3;
 599 my $MAX_NUMA = 8;
 600
 601 my $numa_fmt = {
 602     cpus => {
 603         type => "string",
 604         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 605         description => "CPUs accessing this NUMA node.",
 606         format_description => "id[-id];...",
 607     },
 608     memory => {
 609         type => "number",
 610         description => "Amount of memory this NUMA node provides.",
 611         optional => 1,
 612     },
 613     hostnodes => {
 614         type => "string",
 615         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 616         description => "Host NUMA nodes to use.",
 617         format_description => "id[-id];...",
 618         optional => 1,
 619     },
 620     policy => {
 621         type => 'string',
 622         enum => [qw(preferred bind interleave)],
 623         description => "NUMA allocation policy.",
 624         optional => 1,
 625     },
 626 };
 627 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 628 my $numadesc = {
 629     optional => 1,
 630     type => 'string', format => $numa_fmt,
 631     description => "NUMA topology.",
 632 };
 633 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 634
 635 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 636     $confdesc->{"numa$i"} = $numadesc;
 637 }
 638
 639 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 640                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 641                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 642 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 643
 644 my $net_fmt_bridge_descr = <<__EOD__;
 645 Bridge to attach the network device to. The Proxmox VE standard bridge
 646 is called 'vmbr0'.
 647
 648 If you do not specify a bridge, we create a kvm user (NATed) network
 649 device, which provides DHCP and DNS services. The following addresses
 650 are used:
 651
 652  10.0.2.2   Gateway
 653  10.0.2.3   DNS Server
 654  10.0.2.4   SMB Server
 655
 656 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 657 __EOD__
 658
 659 my $net_fmt = {
 660     macaddr => {
 661         type => 'string',
 662         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 663         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 664         format_description => "XX:XX:XX:XX:XX:XX",
 665         optional => 1,
 666     },
 667     model => {
 668         type => 'string',
 669         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 670         enum => $nic_model_list,
 671         default_key => 1,
 672     },
 673     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 674     bridge => {
 675         type => 'string',
 676         description => $net_fmt_bridge_descr,
 677         format_description => 'bridge',
 678         optional => 1,
 679     },
 680     queues => {
 681         type => 'integer',
 682         minimum => 0, maximum => 16,
 683         description => 'Number of packet queues to be used on the device.',
 684         optional => 1,
 685     },
 686     rate => {
 687         type => 'number',
 688         minimum => 0,
 689         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 690         optional => 1,
 691     },
 692     tag => {
 693         type => 'integer',
 694         minimum => 1, maximum => 4094,
 695         description => 'VLAN tag to apply to packets on this interface.',
 696         optional => 1,
 697     },
 698     trunks => {
 699         type => 'string',
 700         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 701         description => 'VLAN trunks to pass through this interface.',
 702         format_description => 'vlanid[;vlanid...]',
 703         optional => 1,
 704     },
 705     firewall => {
 706         type => 'boolean',
 707         description => 'Whether this interface should be protected by the firewall.',
 708         optional => 1,
 709     },
 710     link_down => {
 711         type => 'boolean',
 712         description => 'Whether this interface should be disconnected (like pulling the plug).',
 713         optional => 1,
 714     },
 715 };
 716
 717 my $netdesc = {
 718     optional => 1,
 719     type => 'string', format => $net_fmt,
 720     description => "Specify network devices.",
 721 };
 722
 723 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 724
 725 my $ipconfig_fmt = {
 726     ip => {
 727         type => 'string',
 728         format => 'pve-ipv4-config',
 729         format_description => 'IPv4Format/CIDR',
 730         description => 'IPv4 address in CIDR format.',
 731         optional => 1,
 732         default => 'dhcp',
 733     },
 734     gw => {
 735         type => 'string',
 736         format => 'ipv4',
 737         format_description => 'GatewayIPv4',
 738         description => 'Default gateway for IPv4 traffic.',
 739         optional => 1,
 740         requires => 'ip',
 741     },
 742     ip6 => {
 743         type => 'string',
 744         format => 'pve-ipv6-config',
 745         format_description => 'IPv6Format/CIDR',
 746         description => 'IPv6 address in CIDR format.',
 747         optional => 1,
 748         default => 'dhcp',
 749     },
 750     gw6 => {
 751         type => 'string',
 752         format => 'ipv6',
 753         format_description => 'GatewayIPv6',
 754         description => 'Default gateway for IPv6 traffic.',
 755         optional => 1,
 756         requires => 'ip6',
 757     },
 758 };
 759 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 760 my $ipconfigdesc = {
 761     optional => 1,
 762     type => 'string', format => 'pve-qm-ipconfig',
 763     description => <<'EODESCR',
 764 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 765
 766 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 767
 768 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 769 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 770
 771 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 772 EODESCR
 773 };
 774 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 775
 776 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 777     $confdesc->{"net$i"} = $netdesc;
 778     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 779 }
 780
 781 foreach my $key (keys %$confdesc_cloudinit) {
 782     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 783 }
 784
 785 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 786 sub verify_volume_id_or_qm_path {
 787     my ($volid, $noerr) = @_;
 788
 789     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 790         return $volid;
 791     }
 792
 793     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 794     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 795     if ($@) {
 796         return undef if $noerr;
 797         die $@;
 798     }
 799     return $volid;
 800 }
 801
 802 my $drivename_hash;
 803
 804 my %drivedesc_base = (
 805     volume => { alias => 'file' },
 806     file => {
 807         type => 'string',
 808         format => 'pve-volume-id-or-qm-path',
 809         default_key => 1,
 810         format_description => 'volume',
 811         description => "The drive's backing volume.",
 812     },
 813     media => {
 814         type => 'string',
 815         enum => [qw(cdrom disk)],
 816         description => "The drive's media type.",
 817         default => 'disk',
 818         optional => 1
 819     },
 820     cyls => {
 821         type => 'integer',
 822         description => "Force the drive's physical geometry to have a specific cylinder count.",
 823         optional => 1
 824     },
 825     heads => {
 826         type => 'integer',
 827         description => "Force the drive's physical geometry to have a specific head count.",
 828         optional => 1
 829     },
 830     secs => {
 831         type => 'integer',
 832         description => "Force the drive's physical geometry to have a specific sector count.",
 833         optional => 1
 834     },
 835     trans => {
 836         type => 'string',
 837         enum => [qw(none lba auto)],
 838         description => "Force disk geometry bios translation mode.",
 839         optional => 1,
 840     },
 841     snapshot => {
 842         type => 'boolean',
 843         description => "Controls qemu's snapshot mode feature."
 844             . " If activated, changes made to the disk are temporary and will"
 845             . " be discarded when the VM is shutdown.",
 846         optional => 1,
 847     },
 848     cache => {
 849         type => 'string',
 850         enum => [qw(none writethrough writeback unsafe directsync)],
 851         description => "The drive's cache mode",
 852         optional => 1,
 853     },
 854     format => get_standard_option('pve-qm-image-format'),
 855     size => {
 856         type => 'string',
 857         format => 'disk-size',
 858         format_description => 'DiskSize',
 859         description => "Disk size. This is purely informational and has no effect.",
 860         optional => 1,
 861     },
 862     backup => {
 863         type => 'boolean',
 864         description => "Whether the drive should be included when making backups.",
 865         optional => 1,
 866     },
 867     replicate => {
 868         type => 'boolean',
 869         description => 'Whether the drive should considered for replication jobs.',
 870         optional => 1,
 871         default => 1,
 872     },
 873     rerror => {
 874         type => 'string',
 875         enum => [qw(ignore report stop)],
 876         description => 'Read error action.',
 877         optional => 1,
 878     },
 879     werror => {
 880         type => 'string',
 881         enum => [qw(enospc ignore report stop)],
 882         description => 'Write error action.',
 883         optional => 1,
 884     },
 885     aio => {
 886         type => 'string',
 887         enum => [qw(native threads)],
 888         description => 'AIO type to use.',
 889         optional => 1,
 890     },
 891     discard => {
 892         type => 'string',
 893         enum => [qw(ignore on)],
 894         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 895         optional => 1,
 896     },
 897     detect_zeroes => {
 898         type => 'boolean',
 899         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 900         optional => 1,
 901     },
 902     serial => {
 903         type => 'string',
 904         format => 'urlencoded',
 905         format_description => 'serial',
 906         maxLength => 20*3, # *3 since it's %xx url enoded
 907         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 908         optional => 1,
 909     },
 910     shared => {
 911         type => 'boolean',
 912         description => 'Mark this locally-managed volume as available on all nodes',
 913         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 914         optional => 1,
 915         default => 0,
 916     }
 917 );
 918
 919 my %iothread_fmt = ( iothread => {
 920         type => 'boolean',
 921         description => "Whether to use iothreads for this drive",
 922         optional => 1,
 923 });
 924
 925 my %model_fmt = (
 926     model => {
 927         type => 'string',
 928         format => 'urlencoded',
 929         format_description => 'model',
 930         maxLength => 40*3, # *3 since it's %xx url enoded
 931         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 932         optional => 1,
 933     },
 934 );
 935
 936 my %queues_fmt = (
 937     queues => {
 938         type => 'integer',
 939         description => "Number of queues.",
 940         minimum => 2,
 941         optional => 1
 942     }
 943 );
 944
 945 my %scsiblock_fmt = (
 946     scsiblock => {
 947         type => 'boolean',
 948         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 949         optional => 1,
 950         default => 0,
 951     },
 952 );
 953
 954 my $add_throttle_desc = sub {
 955     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 956     my $d = {
 957         type => $type,
 958         format_description => $unit,
 959         description => "Maximum $what in $longunit.",
 960         optional => 1,
 961     };
 962     $d->{minimum} = $minimum if defined($minimum);
 963     $drivedesc_base{$key} = $d;
 964 };
 965 # throughput: (leaky bucket)
 966 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 967 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 968 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 969 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 970 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 971 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 972 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 973 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 974 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 975
 976 # pools: (pool of IO before throttling starts taking effect)
 977 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 978 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 979 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 980 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 981 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 982 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 983
 984 # burst lengths
 985 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 986 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 987 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 988 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 989 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 990 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 991
 992 # legacy support
 993 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 994 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 995 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 996 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 997
 998 my $ide_fmt = {
 999     %drivedesc_base,
1000     %model_fmt,
1001 };
1002 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1003
1004 my $idedesc = {
1005     optional => 1,
1006     type => 'string', format => $ide_fmt,
1007     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1008 };
1009 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1010
1011 my $scsi_fmt = {
1012     %drivedesc_base,
1013     %iothread_fmt,
1014     %queues_fmt,
1015     %scsiblock_fmt,
1016 };
1017 my $scsidesc = {
1018     optional => 1,
1019     type => 'string', format => $scsi_fmt,
1020     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1021 };
1022 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1023
1024 my $sata_fmt = {
1025     %drivedesc_base,
1026 };
1027 my $satadesc = {
1028     optional => 1,
1029     type => 'string', format => $sata_fmt,
1030     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1031 };
1032 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1033
1034 my $virtio_fmt = {
1035     %drivedesc_base,
1036     %iothread_fmt,
1037 };
1038 my $virtiodesc = {
1039     optional => 1,
1040     type => 'string', format => $virtio_fmt,
1041     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1044
1045 my $alldrive_fmt = {
1046     %drivedesc_base,
1047     %iothread_fmt,
1048     %model_fmt,
1049     %queues_fmt,
1050     %scsiblock_fmt,
1051 };
1052
1053 my $efidisk_fmt = {
1054     volume => { alias => 'file' },
1055     file => {
1056         type => 'string',
1057         format => 'pve-volume-id-or-qm-path',
1058         default_key => 1,
1059         format_description => 'volume',
1060         description => "The drive's backing volume.",
1061     },
1062     format => get_standard_option('pve-qm-image-format'),
1063     size => {
1064         type => 'string',
1065         format => 'disk-size',
1066         format_description => 'DiskSize',
1067         description => "Disk size. This is purely informational and has no effect.",
1068         optional => 1,
1069     },
1070 };
1071
1072 my $efidisk_desc = {
1073     optional => 1,
1074     type => 'string', format => $efidisk_fmt,
1075     description => "Configure a Disk for storing EFI vars",
1076 };
1077
1078 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1079
1080 my $usb_fmt = {
1081     host => {
1082         default_key => 1,
1083         type => 'string', format => 'pve-qm-usb-device',
1084         format_description => 'HOSTUSBDEVICE|spice',
1085         description => <<EODESCR,
1086 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1087
1088  'bus-port(.port)*' (decimal numbers) or
1089  'vendor_id:product_id' (hexadeciaml numbers) or
1090  'spice'
1091
1092 You can use the 'lsusb -t' command to list existing usb devices.
1093
1094 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1095
1096 The value 'spice' can be used to add a usb redirection devices for spice.
1097 EODESCR
1098     },
1099     usb3 => {
1100         optional => 1,
1101         type => 'boolean',
1102         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1103         default => 0,
1104     },
1105 };
1106
1107 my $usbdesc = {
1108     optional => 1,
1109     type => 'string', format => $usb_fmt,
1110     description => "Configure an USB device (n is 0 to 4).",
1111 };
1112 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1113
1114 # NOTE: the match-groups of this regex are used in parse_hostpci
1115 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1116 my $hostpci_fmt = {
1117     host => {
1118         default_key => 1,
1119         type => 'string',
1120         pattern => qr/$PCIRE(;$PCIRE)*/,
1121         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1122         description => <<EODESCR,
1123 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1124 of PCI virtual functions of the host. HOSTPCIID syntax is:
1125
1126 'bus:dev.func' (hexadecimal numbers)
1127
1128 You can us the 'lspci' command to list existing PCI devices.
1129 EODESCR
1130     },
1131     rombar => {
1132         type => 'boolean',
1133         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1134         optional => 1,
1135         default => 1,
1136     },
1137     romfile => {
1138         type => 'string',
1139         pattern => '[^,;]+',
1140         format_description => 'string',
1141         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1142         optional => 1,
1143     },
1144     pcie => {
1145         type => 'boolean',
1146         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1147         optional => 1,
1148         default => 0,
1149     },
1150     'x-vga' => {
1151         type => 'boolean',
1152         description =>  "Enable vfio-vga device support.",
1153         optional => 1,
1154         default => 0,
1155     },
1156 };
1157 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1158
1159 my $hostpcidesc = {
1160         optional => 1,
1161         type => 'string', format => 'pve-qm-hostpci',
1162         description => "Map host PCI devices into guest.",
1163         verbose_description =>  <<EODESCR,
1164 Map host PCI devices into guest.
1165
1166 NOTE: This option allows direct access to host hardware. So it is no longer
1167 possible to migrate such machines - use with special care.
1168
1169 CAUTION: Experimental! User reported problems with this option.
1170 EODESCR
1171 };
1172 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1173
1174 my $serialdesc = {
1175         optional => 1,
1176         type => 'string',
1177         pattern => '(/dev/.+|socket)',
1178         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1179         verbose_description =>  <<EODESCR,
1180 Create a serial device inside the VM (n is 0 to 3), and pass through a
1181 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1182 host side (use 'qm terminal' to open a terminal connection).
1183
1184 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1185
1186 CAUTION: Experimental! User reported problems with this option.
1187 EODESCR
1188 };
1189
1190 my $paralleldesc= {
1191         optional => 1,
1192         type => 'string',
1193         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1194         description =>  "Map host parallel devices (n is 0 to 2).",
1195         verbose_description =>  <<EODESCR,
1196 Map host parallel devices (n is 0 to 2).
1197
1198 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1199
1200 CAUTION: Experimental! User reported problems with this option.
1201 EODESCR
1202 };
1203
1204 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1205     $confdesc->{"parallel$i"} = $paralleldesc;
1206 }
1207
1208 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1209     $confdesc->{"serial$i"} = $serialdesc;
1210 }
1211
1212 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1213     $confdesc->{"hostpci$i"} = $hostpcidesc;
1214 }
1215
1216 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1217     $drivename_hash->{"ide$i"} = 1;
1218     $confdesc->{"ide$i"} = $idedesc;
1219 }
1220
1221 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1222     $drivename_hash->{"sata$i"} = 1;
1223     $confdesc->{"sata$i"} = $satadesc;
1224 }
1225
1226 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1227     $drivename_hash->{"scsi$i"} = 1;
1228     $confdesc->{"scsi$i"} = $scsidesc ;
1229 }
1230
1231 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1232     $drivename_hash->{"virtio$i"} = 1;
1233     $confdesc->{"virtio$i"} = $virtiodesc;
1234 }
1235
1236 $drivename_hash->{efidisk0} = 1;
1237 $confdesc->{efidisk0} = $efidisk_desc;
1238
1239 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1240     $confdesc->{"usb$i"} = $usbdesc;
1241 }
1242
1243 my $unuseddesc = {
1244     optional => 1,
1245     type => 'string', format => 'pve-volume-id',
1246     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1247 };
1248
1249 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1250     $confdesc->{"unused$i"} = $unuseddesc;
1251 }
1252
1253 my $kvm_api_version = 0;
1254
1255 sub kvm_version {
1256
1257     return $kvm_api_version if $kvm_api_version;
1258
1259     my $fh = IO::File->new("</dev/kvm") ||
1260         return 0;
1261
1262     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1263         $kvm_api_version = $v;
1264     }
1265
1266     $fh->close();
1267
1268     return  $kvm_api_version;
1269 }
1270
1271 my $kvm_user_version;
1272
1273 sub kvm_user_version {
1274
1275     return $kvm_user_version if $kvm_user_version;
1276
1277     $kvm_user_version = 'unknown';
1278
1279     my $code = sub {
1280         my $line = shift;
1281         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1282             $kvm_user_version = $2;
1283         }
1284     };
1285
1286     eval { run_command("kvm -version", outfunc => $code); };
1287     warn $@ if $@;
1288
1289     return $kvm_user_version;
1290
1291 }
1292
1293 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1294
1295 sub valid_drive_names {
1296     # order is important - used to autoselect boot disk
1297     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1298             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1299             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1300             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1301             'efidisk0');
1302 }
1303
1304 sub is_valid_drivename {
1305     my $dev = shift;
1306
1307     return defined($drivename_hash->{$dev});
1308 }
1309
1310 sub option_exists {
1311     my $key = shift;
1312     return defined($confdesc->{$key});
1313 }
1314
1315 sub nic_models {
1316     return $nic_model_list;
1317 }
1318
1319 sub os_list_description {
1320
1321     return {
1322         other => 'Other',
1323         wxp => 'Windows XP',
1324         w2k => 'Windows 2000',
1325         w2k3 =>, 'Windows 2003',
1326         w2k8 => 'Windows 2008',
1327         wvista => 'Windows Vista',
1328         win7 => 'Windows 7',
1329         win8 => 'Windows 8/2012',
1330         win10 => 'Windows 10/2016',
1331         l24 => 'Linux 2.4',
1332         l26 => 'Linux 2.6',
1333     };
1334 }
1335
1336 my $cdrom_path;
1337
1338 sub get_cdrom_path {
1339
1340     return  $cdrom_path if $cdrom_path;
1341
1342     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1343     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1344     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1345 }
1346
1347 sub get_iso_path {
1348     my ($storecfg, $vmid, $cdrom) = @_;
1349
1350     if ($cdrom eq 'cdrom') {
1351         return get_cdrom_path();
1352     } elsif ($cdrom eq 'none') {
1353         return '';
1354     } elsif ($cdrom =~ m|^/|) {
1355         return $cdrom;
1356     } else {
1357         return PVE::Storage::path($storecfg, $cdrom);
1358     }
1359 }
1360
1361 # try to convert old style file names to volume IDs
1362 sub filename_to_volume_id {
1363     my ($vmid, $file, $media) = @_;
1364
1365      if (!($file eq 'none' || $file eq 'cdrom' ||
1366           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1367
1368         return undef if $file =~ m|/|;
1369
1370         if ($media && $media eq 'cdrom') {
1371             $file = "local:iso/$file";
1372         } else {
1373             $file = "local:$vmid/$file";
1374         }
1375     }
1376
1377     return $file;
1378 }
1379
1380 sub verify_media_type {
1381     my ($opt, $vtype, $media) = @_;
1382
1383     return if !$media;
1384
1385     my $etype;
1386     if ($media eq 'disk') {
1387         $etype = 'images';
1388     } elsif ($media eq 'cdrom') {
1389         $etype = 'iso';
1390     } else {
1391         die "internal error";
1392     }
1393
1394     return if ($vtype eq $etype);
1395
1396     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1397 }
1398
1399 sub cleanup_drive_path {
1400     my ($opt, $storecfg, $drive) = @_;
1401
1402     # try to convert filesystem paths to volume IDs
1403
1404     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1405         ($drive->{file} !~ m|^/dev/.+|) &&
1406         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1407         ($drive->{file} !~ m/^\d+$/)) {
1408         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1409         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1410         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1411         verify_media_type($opt, $vtype, $drive->{media});
1412         $drive->{file} = $volid;
1413     }
1414
1415     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1416 }
1417
1418 sub parse_hotplug_features {
1419     my ($data) = @_;
1420
1421     my $res = {};
1422
1423     return $res if $data eq '0';
1424
1425     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1426
1427     foreach my $feature (PVE::Tools::split_list($data)) {
1428         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1429             $res->{$1} = 1;
1430         } else {
1431             die "invalid hotplug feature '$feature'\n";
1432         }
1433     }
1434     return $res;
1435 }
1436
1437 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1438 sub pve_verify_hotplug_features {
1439     my ($value, $noerr) = @_;
1440
1441     return $value if parse_hotplug_features($value);
1442
1443     return undef if $noerr;
1444
1445     die "unable to parse hotplug option\n";
1446 }
1447
1448 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1449 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1450 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1451 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1452 #        [,iothread=on][,serial=serial][,model=model]
1453
1454 sub parse_drive {
1455     my ($key, $data) = @_;
1456
1457     my ($interface, $index);
1458
1459     if ($key =~ m/^([^\d]+)(\d+)$/) {
1460         $interface = $1;
1461         $index = $2;
1462     } else {
1463         return undef;
1464     }
1465
1466     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1467                                      : $confdesc->{$key}->{format};
1468     if (!$desc) {
1469         warn "invalid drive key: $key\n";
1470         return undef;
1471     }
1472     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1473     return undef if !$res;
1474     $res->{interface} = $interface;
1475     $res->{index} = $index;
1476
1477     my $error = 0;
1478     foreach my $opt (qw(bps bps_rd bps_wr)) {
1479         if (my $bps = defined(delete $res->{$opt})) {
1480             if (defined($res->{"m$opt"})) {
1481                 warn "both $opt and m$opt specified\n";
1482                 ++$error;
1483                 next;
1484             }
1485             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1486         }
1487     }
1488
1489     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1490     for my $requirement (
1491         [mbps_max => 'mbps'],
1492         [mbps_rd_max => 'mbps_rd'],
1493         [mbps_wr_max => 'mbps_wr'],
1494         [miops_max => 'miops'],
1495         [miops_rd_max => 'miops_rd'],
1496         [miops_wr_max => 'miops_wr'],
1497         [bps_max_length => 'mbps_max'],
1498         [bps_rd_max_length => 'mbps_rd_max'],
1499         [bps_wr_max_length => 'mbps_wr_max'],
1500         [iops_max_length => 'iops_max'],
1501         [iops_rd_max_length => 'iops_rd_max'],
1502         [iops_wr_max_length => 'iops_wr_max']) {
1503         my ($option, $requires) = @$requirement;
1504         if ($res->{$option} && !$res->{$requires}) {
1505             warn "$option requires $requires\n";
1506             ++$error;
1507         }
1508     }
1509
1510     return undef if $error;
1511
1512     return undef if $res->{mbps_rd} && $res->{mbps};
1513     return undef if $res->{mbps_wr} && $res->{mbps};
1514     return undef if $res->{iops_rd} && $res->{iops};
1515     return undef if $res->{iops_wr} && $res->{iops};
1516
1517     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1518         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1519         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1520         return undef if $res->{interface} eq 'virtio';
1521     }
1522
1523     if (my $size = $res->{size}) {
1524         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1525     }
1526
1527     return $res;
1528 }
1529
1530 sub print_drive {
1531     my ($vmid, $drive) = @_;
1532     my $data = { %$drive };
1533     delete $data->{$_} for qw(index interface);
1534     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1535 }
1536
1537 sub scsi_inquiry {
1538     my($fh, $noerr) = @_;
1539
1540     my $SG_IO = 0x2285;
1541     my $SG_GET_VERSION_NUM = 0x2282;
1542
1543     my $versionbuf = "\x00" x 8;
1544     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1545     if (!$ret) {
1546         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1547         return undef;
1548     }
1549     my $version = unpack("I", $versionbuf);
1550     if ($version < 30000) {
1551         die "scsi generic interface too old\n"  if !$noerr;
1552         return undef;
1553     }
1554
1555     my $buf = "\x00" x 36;
1556     my $sensebuf = "\x00" x 8;
1557     my $cmd = pack("C x3 C x1", 0x12, 36);
1558
1559     # see /usr/include/scsi/sg.h
1560     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1561
1562     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1563                       length($sensebuf), 0, length($buf), $buf,
1564                       $cmd, $sensebuf, 6000);
1565
1566     $ret = ioctl($fh, $SG_IO, $packet);
1567     if (!$ret) {
1568         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1569         return undef;
1570     }
1571
1572     my @res = unpack($sg_io_hdr_t, $packet);
1573     if ($res[17] || $res[18]) {
1574         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1575         return undef;
1576     }
1577
1578     my $res = {};
1579     (my $byte0, my $byte1, $res->{vendor},
1580      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1581
1582     $res->{removable} = $byte1 & 128 ? 1 : 0;
1583     $res->{type} = $byte0 & 31;
1584
1585     return $res;
1586 }
1587
1588 sub path_is_scsi {
1589     my ($path) = @_;
1590
1591     my $fh = IO::File->new("+<$path") || return undef;
1592     my $res = scsi_inquiry($fh, 1);
1593     close($fh);
1594
1595     return $res;
1596 }
1597
1598 sub machine_type_is_q35 {
1599     my ($conf) = @_;
1600
1601     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1602 }
1603
1604 sub print_tabletdevice_full {
1605     my ($conf) = @_;
1606
1607     my $q35 = machine_type_is_q35($conf);
1608
1609     # we use uhci for old VMs because tablet driver was buggy in older qemu
1610     my $usbbus = $q35 ? "ehci" : "uhci";
1611
1612     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1613 }
1614
1615 sub print_drivedevice_full {
1616     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1617
1618     my $device = '';
1619     my $maxdev = 0;
1620
1621     if ($drive->{interface} eq 'virtio') {
1622         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1623         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1624         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1625     } elsif ($drive->{interface} eq 'scsi') {
1626
1627         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1628         my $unit = $drive->{index} % $maxdev;
1629         my $devicetype = 'hd';
1630         my $path = '';
1631         if (drive_is_cdrom($drive)) {
1632             $devicetype = 'cd';
1633         } else {
1634             if ($drive->{file} =~ m|^/|) {
1635                 $path = $drive->{file};
1636                 if (my $info = path_is_scsi($path)) {
1637                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1638                         $devicetype = 'block';
1639                     } elsif ($info->{type} == 1) { # tape
1640                         $devicetype = 'generic';
1641                     }
1642                 }
1643             } else {
1644                  $path = PVE::Storage::path($storecfg, $drive->{file});
1645             }
1646
1647             if($path =~ m/^iscsi\:\/\//){
1648                 $devicetype = 'generic';
1649             }
1650         }
1651
1652         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1653             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1654         } else {
1655             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1656         }
1657
1658     } elsif ($drive->{interface} eq 'ide'){
1659         $maxdev = 2;
1660         my $controller = int($drive->{index} / $maxdev);
1661         my $unit = $drive->{index} % $maxdev;
1662         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1663
1664         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1666             $model = URI::Escape::uri_unescape($model);
1667             $device .= ",model=$model";
1668         }
1669     } elsif ($drive->{interface} eq 'sata'){
1670         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1671         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1672         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1673     } elsif ($drive->{interface} eq 'usb') {
1674         die "implement me";
1675         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1676     } else {
1677         die "unsupported interface type";
1678     }
1679
1680     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1681
1682     return $device;
1683 }
1684
1685 sub get_initiator_name {
1686     my $initiator;
1687
1688     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1689     while (defined(my $line = <$fh>)) {
1690         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1691         $initiator = $1;
1692         last;
1693     }
1694     $fh->close();
1695
1696     return $initiator;
1697 }
1698
1699 sub print_drive_full {
1700     my ($storecfg, $vmid, $drive) = @_;
1701
1702     my $path;
1703     my $volid = $drive->{file};
1704     my $format;
1705
1706     if (drive_is_cdrom($drive)) {
1707         $path = get_iso_path($storecfg, $vmid, $volid);
1708     } else {
1709         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1710         if ($storeid) {
1711             $path = PVE::Storage::path($storecfg, $volid);
1712             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1713             $format = qemu_img_format($scfg, $volname);
1714         } else {
1715             $path = $volid;
1716             $format = "raw";
1717         }
1718    }
1719
1720     my $opts = '';
1721     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1722     foreach my $o (@qemu_drive_options) {
1723         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1724     }
1725
1726     # snapshot only accepts on|off
1727     if (defined($drive->{snapshot})) {
1728         my $v = $drive->{snapshot} ? 'on' : 'off';
1729         $opts .= ",snapshot=$v";
1730     }
1731
1732     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1733         my ($dir, $qmpname) = @$type;
1734         if (my $v = $drive->{"mbps$dir"}) {
1735             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1736         }
1737         if (my $v = $drive->{"mbps${dir}_max"}) {
1738             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1739         }
1740         if (my $v = $drive->{"bps${dir}_max_length"}) {
1741             $opts .= ",throttling.bps$qmpname-max-length=$v";
1742         }
1743         if (my $v = $drive->{"iops${dir}"}) {
1744             $opts .= ",throttling.iops$qmpname=$v";
1745         }
1746         if (my $v = $drive->{"iops${dir}_max"}) {
1747             $opts .= ",throttling.iops$qmpname-max=$v";
1748         }
1749         if (my $v = $drive->{"iops${dir}_max_length"}) {
1750             $opts .= ",throttling.iops$qmpname-max-length=$v";
1751         }
1752     }
1753
1754     if (my $serial = $drive->{serial}) {
1755         $serial = URI::Escape::uri_unescape($serial);
1756         $opts .= ",serial=$serial";
1757     }
1758
1759     $opts .= ",format=$format" if $format && !$drive->{format};
1760
1761     my $cache_direct = 0;
1762
1763     if (my $cache = $drive->{cache}) {
1764         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1765     } elsif (!drive_is_cdrom($drive)) {
1766         $opts .= ",cache=none";
1767         $cache_direct = 1;
1768     }
1769
1770     # aio native works only with O_DIRECT
1771     if (!$drive->{aio}) {
1772         if($cache_direct) {
1773             $opts .= ",aio=native";
1774         } else {
1775             $opts .= ",aio=threads";
1776         }
1777     }
1778
1779     if (!drive_is_cdrom($drive)) {
1780         my $detectzeroes;
1781         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1782             $detectzeroes = 'off';
1783         } elsif ($drive->{discard}) {
1784             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1785         } else {
1786             # This used to be our default with discard not being specified:
1787             $detectzeroes = 'on';
1788         }
1789         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1790     }
1791
1792     my $pathinfo = $path ? "file=$path," : '';
1793
1794     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1795 }
1796
1797 sub print_netdevice_full {
1798     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1799
1800     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1801
1802     my $device = $net->{model};
1803     if ($net->{model} eq 'virtio') {
1804          $device = 'virtio-net-pci';
1805      };
1806
1807     my $pciaddr = print_pci_addr("$netid", $bridges);
1808     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1809     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1810         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1811         my $vectors = $net->{queues} * 2 + 2;
1812         $tmpstr .= ",vectors=$vectors,mq=on";
1813     }
1814     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1815
1816     if ($use_old_bios_files) {
1817         my $romfile;
1818         if ($device eq 'virtio-net-pci') {
1819             $romfile = 'pxe-virtio.rom';
1820         } elsif ($device eq 'e1000') {
1821             $romfile = 'pxe-e1000.rom';
1822         } elsif ($device eq 'ne2k') {
1823             $romfile = 'pxe-ne2k_pci.rom';
1824         } elsif ($device eq 'pcnet') {
1825             $romfile = 'pxe-pcnet.rom';
1826         } elsif ($device eq 'rtl8139') {
1827             $romfile = 'pxe-rtl8139.rom';
1828         }
1829         $tmpstr .= ",romfile=$romfile" if $romfile;
1830     }
1831
1832     return $tmpstr;
1833 }
1834
1835 sub print_netdev_full {
1836     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1837
1838     my $i = '';
1839     if ($netid =~ m/^net(\d+)$/) {
1840         $i = int($1);
1841     }
1842
1843     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1844
1845     my $ifname = "tap${vmid}i$i";
1846
1847     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1848     die "interface name '$ifname' is too long (max 15 character)\n"
1849         if length($ifname) >= 16;
1850
1851     my $vhostparam = '';
1852     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1853
1854     my $vmname = $conf->{name} || "vm$vmid";
1855
1856     my $netdev = "";
1857     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1858
1859     if ($net->{bridge}) {
1860         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1861     } else {
1862         $netdev = "type=user,id=$netid,hostname=$vmname";
1863     }
1864
1865     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1866
1867     return $netdev;
1868 }
1869
1870
1871 sub print_cpu_device {
1872     my ($conf, $id) = @_;
1873
1874     my $kvm = $conf->{kvm} // 1;
1875     my $cpu = $kvm ? "kvm64" : "qemu64";
1876     if (my $cputype = $conf->{cpu}) {
1877         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1878             or die "Cannot parse cpu description: $cputype\n";
1879         $cpu = $cpuconf->{cputype};
1880     }
1881
1882     my $cores = $conf->{cores} || 1;
1883
1884     my $current_core = ($id - 1) % $cores;
1885     my $current_socket = int(($id - 1 - $current_core)/$cores);
1886
1887     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1888 }
1889
1890 sub drive_is_cloudinit {
1891     my ($drive) = @_;
1892     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1893 }
1894
1895 sub drive_is_cdrom {
1896     my ($drive, $exclude_cloudinit) = @_;
1897
1898     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1899
1900     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1901
1902 }
1903
1904 sub parse_number_sets {
1905     my ($set) = @_;
1906     my $res = [];
1907     foreach my $part (split(/;/, $set)) {
1908         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1909             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1910             push @$res, [ $1, $2 ];
1911         } else {
1912             die "invalid range: $part\n";
1913         }
1914     }
1915     return $res;
1916 }
1917
1918 sub parse_numa {
1919     my ($data) = @_;
1920
1921     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1922     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1923     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1924     return $res;
1925 }
1926
1927 sub parse_hostpci {
1928     my ($value) = @_;
1929
1930     return undef if !$value;
1931
1932     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1933
1934     my @idlist = split(/;/, $res->{host});
1935     delete $res->{host};
1936     foreach my $id (@idlist) {
1937         if ($id =~ /^$PCIRE$/) {
1938             if (defined($2)) {
1939                 push @{$res->{pciid}}, { id => $1, function => $2 };
1940             } else {
1941                 my $pcidevices = lspci($1);
1942                 $res->{pciid} = $pcidevices->{$1};
1943             }
1944         } else {
1945             # should have been caught by parse_property_string already
1946             die "failed to parse PCI id: $id\n";
1947         }
1948     }
1949     return $res;
1950 }
1951
1952 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1953 sub parse_net {
1954     my ($data) = @_;
1955
1956     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1957     if ($@) {
1958         warn $@;
1959         return undef;
1960     }
1961     if (!defined($res->{macaddr})) {
1962         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1963         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1964     }
1965     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1966     return $res;
1967 }
1968
1969 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1970 sub parse_ipconfig {
1971     my ($data) = @_;
1972
1973     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1974     if ($@) {
1975         warn $@;
1976         return undef;
1977     }
1978
1979     if ($res->{gw} && !$res->{ip}) {
1980         warn 'gateway specified without specifying an IP address';
1981         return undef;
1982     }
1983     if ($res->{gw6} && !$res->{ip6}) {
1984         warn 'IPv6 gateway specified without specifying an IPv6 address';
1985         return undef;
1986     }
1987     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1988         warn 'gateway specified together with DHCP';
1989         return undef;
1990     }
1991     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1992         # gw6 + auto/dhcp
1993         warn "IPv6 gateway specified together with $res->{ip6} address";
1994         return undef;
1995     }
1996
1997     if (!$res->{ip} && !$res->{ip6}) {
1998         return { ip => 'dhcp', ip6 => 'dhcp' };
1999     }
2000
2001     return $res;
2002 }
2003
2004 sub print_net {
2005     my $net = shift;
2006
2007     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2008 }
2009
2010 sub add_random_macs {
2011     my ($settings) = @_;
2012
2013     foreach my $opt (keys %$settings) {
2014         next if $opt !~ m/^net(\d+)$/;
2015         my $net = parse_net($settings->{$opt});
2016         next if !$net;
2017         $settings->{$opt} = print_net($net);
2018     }
2019 }
2020
2021 sub vm_is_volid_owner {
2022     my ($storecfg, $vmid, $volid) = @_;
2023
2024     if ($volid !~  m|^/|) {
2025         my ($path, $owner);
2026         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2027         if ($owner && ($owner == $vmid)) {
2028             return 1;
2029         }
2030     }
2031
2032     return undef;
2033 }
2034
2035 sub split_flagged_list {
2036     my $text = shift || '';
2037     $text =~ s/[,;]/ /g;
2038     $text =~ s/^\s+//;
2039     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2040 }
2041
2042 sub join_flagged_list {
2043     my ($how, $lst) = @_;
2044     join $how, map { $lst->{$_} . $_ } keys %$lst;
2045 }
2046
2047 sub vmconfig_delete_pending_option {
2048     my ($conf, $key, $force) = @_;
2049
2050     delete $conf->{pending}->{$key};
2051     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2052     $pending_delete_hash->{$key} = $force ? '!' : '';
2053     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2054 }
2055
2056 sub vmconfig_undelete_pending_option {
2057     my ($conf, $key) = @_;
2058
2059     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2060     delete $pending_delete_hash->{$key};
2061
2062     if (%$pending_delete_hash) {
2063         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2064     } else {
2065         delete $conf->{pending}->{delete};
2066     }
2067 }
2068
2069 sub vmconfig_register_unused_drive {
2070     my ($storecfg, $vmid, $conf, $drive) = @_;
2071
2072     if (drive_is_cloudinit($drive)) {
2073         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2074         warn $@ if $@;
2075     } elsif (!drive_is_cdrom($drive)) {
2076         my $volid = $drive->{file};
2077         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2078             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2079         }
2080     }
2081 }
2082
2083 sub vmconfig_cleanup_pending {
2084     my ($conf) = @_;
2085
2086     # remove pending changes when nothing changed
2087     my $changes;
2088     foreach my $opt (keys %{$conf->{pending}}) {
2089         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2090             $changes = 1;
2091             delete $conf->{pending}->{$opt};
2092         }
2093     }
2094
2095     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2096     my $pending_delete_hash = {};
2097     while (my ($opt, $force) = each %$current_delete_hash) {
2098         if (defined($conf->{$opt})) {
2099             $pending_delete_hash->{$opt} = $force;
2100         } else {
2101             $changes = 1;
2102         }
2103     }
2104
2105     if (%$pending_delete_hash) {
2106         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2107     } else {
2108         delete $conf->{pending}->{delete};
2109     }
2110
2111     return $changes;
2112 }
2113
2114 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2115 my $smbios1_fmt = {
2116     uuid => {
2117         type => 'string',
2118         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2119         format_description => 'UUID',
2120         description => "Set SMBIOS1 UUID.",
2121         optional => 1,
2122     },
2123     version => {
2124         type => 'string',
2125         pattern => '\S+',
2126         format_description => 'string',
2127         description => "Set SMBIOS1 version.",
2128         optional => 1,
2129     },
2130     serial => {
2131         type => 'string',
2132         pattern => '\S+',
2133         format_description => 'string',
2134         description => "Set SMBIOS1 serial number.",
2135         optional => 1,
2136     },
2137     manufacturer => {
2138         type => 'string',
2139         pattern => '\S+',
2140         format_description => 'string',
2141         description => "Set SMBIOS1 manufacturer.",
2142         optional => 1,
2143     },
2144     product => {
2145         type => 'string',
2146         pattern => '\S+',
2147         format_description => 'string',
2148         description => "Set SMBIOS1 product ID.",
2149         optional => 1,
2150     },
2151     sku => {
2152         type => 'string',
2153         pattern => '\S+',
2154         format_description => 'string',
2155         description => "Set SMBIOS1 SKU string.",
2156         optional => 1,
2157     },
2158     family => {
2159         type => 'string',
2160         pattern => '\S+',
2161         format_description => 'string',
2162         description => "Set SMBIOS1 family string.",
2163         optional => 1,
2164     },
2165 };
2166
2167 sub parse_smbios1 {
2168     my ($data) = @_;
2169
2170     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2171     warn $@ if $@;
2172     return $res;
2173 }
2174
2175 sub print_smbios1 {
2176     my ($smbios1) = @_;
2177     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2178 }
2179
2180 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2181
2182 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2183 sub verify_bootdisk {
2184     my ($value, $noerr) = @_;
2185
2186     return $value if is_valid_drivename($value);
2187
2188     return undef if $noerr;
2189
2190     die "invalid boot disk '$value'\n";
2191 }
2192
2193 sub parse_watchdog {
2194     my ($value) = @_;
2195
2196     return undef if !$value;
2197
2198     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2199     warn $@ if $@;
2200     return $res;
2201 }
2202
2203 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2204 sub verify_usb_device {
2205     my ($value, $noerr) = @_;
2206
2207     return $value if parse_usb_device($value);
2208
2209     return undef if $noerr;
2210
2211     die "unable to parse usb device\n";
2212 }
2213
2214 # add JSON properties for create and set function
2215 sub json_config_properties {
2216     my $prop = shift;
2217
2218     foreach my $opt (keys %$confdesc) {
2219         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2220         $prop->{$opt} = $confdesc->{$opt};
2221     }
2222
2223     return $prop;
2224 }
2225
2226 sub check_type {
2227     my ($key, $value) = @_;
2228
2229     die "unknown setting '$key'\n" if !$confdesc->{$key};
2230
2231     my $type = $confdesc->{$key}->{type};
2232
2233     if (!defined($value)) {
2234         die "got undefined value\n";
2235     }
2236
2237     if ($value =~ m/[\n\r]/) {
2238         die "property contains a line feed\n";
2239     }
2240
2241     if ($type eq 'boolean') {
2242         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2243         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2244         die "type check ('boolean') failed - got '$value'\n";
2245     } elsif ($type eq 'integer') {
2246         return int($1) if $value =~ m/^(\d+)$/;
2247         die "type check ('integer') failed - got '$value'\n";
2248     } elsif ($type eq 'number') {
2249         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2250         die "type check ('number') failed - got '$value'\n";
2251     } elsif ($type eq 'string') {
2252         if (my $fmt = $confdesc->{$key}->{format}) {
2253             PVE::JSONSchema::check_format($fmt, $value);
2254             return $value;
2255         }
2256         $value =~ s/^\"(.*)\"$/$1/;
2257         return $value;
2258     } else {
2259         die "internal error"
2260     }
2261 }
2262
2263 sub check_iommu_support{
2264     #fixme : need to check IOMMU support
2265     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2266
2267     my $iommu=1;
2268     return $iommu;
2269
2270 }
2271
2272 sub touch_config {
2273     my ($vmid) = @_;
2274
2275     my $conf = PVE::QemuConfig->config_file($vmid);
2276     utime undef, undef, $conf;
2277 }
2278
2279 sub destroy_vm {
2280     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2281
2282     my $conffile = PVE::QemuConfig->config_file($vmid);
2283
2284     my $conf = PVE::QemuConfig->load_config($vmid);
2285
2286     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2287
2288     if ($conf->{template}) {
2289         # check if any base image is still used by a linked clone
2290         foreach_drive($conf, sub {
2291                 my ($ds, $drive) = @_;
2292
2293                 return if drive_is_cdrom($drive);
2294
2295                 my $volid = $drive->{file};
2296
2297                 return if !$volid || $volid =~ m|^/|;
2298
2299                 die "base volume '$volid' is still in use by linked cloned\n"
2300                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2301
2302         });
2303     }
2304
2305     # only remove disks owned by this VM
2306     foreach_drive($conf, sub {
2307         my ($ds, $drive) = @_;
2308
2309         return if drive_is_cdrom($drive, 1);
2310
2311         my $volid = $drive->{file};
2312
2313         return if !$volid || $volid =~ m|^/|;
2314
2315         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2316         return if !$path || !$owner || ($owner != $vmid);
2317
2318         eval {
2319             PVE::Storage::vdisk_free($storecfg, $volid);
2320         };
2321         warn "Could not remove disk '$volid', check manually: $@" if $@;
2322
2323     });
2324
2325     if ($keep_empty_config) {
2326         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2327     } else {
2328         unlink $conffile;
2329     }
2330
2331     # also remove unused disk
2332     eval {
2333         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2334
2335         eval {
2336             PVE::Storage::foreach_volid($dl, sub {
2337                 my ($volid, $sid, $volname, $d) = @_;
2338                 PVE::Storage::vdisk_free($storecfg, $volid);
2339             });
2340         };
2341         warn $@ if $@;
2342
2343     };
2344     warn $@ if $@;
2345 }
2346
2347 sub parse_vm_config {
2348     my ($filename, $raw) = @_;
2349
2350     return undef if !defined($raw);
2351
2352     my $res = {
2353         digest => Digest::SHA::sha1_hex($raw),
2354         snapshots => {},
2355         pending => {},
2356     };
2357
2358     $filename =~ m|/qemu-server/(\d+)\.conf$|
2359         || die "got strange filename '$filename'";
2360
2361     my $vmid = $1;
2362
2363     my $conf = $res;
2364     my $descr;
2365     my $section = '';
2366
2367     my @lines = split(/\n/, $raw);
2368     foreach my $line (@lines) {
2369         next if $line =~ m/^\s*$/;
2370
2371         if ($line =~ m/^\[PENDING\]\s*$/i) {
2372             $section = 'pending';
2373             if (defined($descr)) {
2374                 $descr =~ s/\s+$//;
2375                 $conf->{description} = $descr;
2376             }
2377             $descr = undef;
2378             $conf = $res->{$section} = {};
2379             next;
2380
2381         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2382             $section = $1;
2383             if (defined($descr)) {
2384                 $descr =~ s/\s+$//;
2385                 $conf->{description} = $descr;
2386             }
2387             $descr = undef;
2388             $conf = $res->{snapshots}->{$section} = {};
2389             next;
2390         }
2391
2392         if ($line =~ m/^\#(.*)\s*$/) {
2393             $descr = '' if !defined($descr);
2394             $descr .= PVE::Tools::decode_text($1) . "\n";
2395             next;
2396         }
2397
2398         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2399             $descr = '' if !defined($descr);
2400             $descr .= PVE::Tools::decode_text($2);
2401         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2402             $conf->{snapstate} = $1;
2403         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2404             my $key = $1;
2405             my $value = $2;
2406             $conf->{$key} = $value;
2407         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2408             my $value = $1;
2409             if ($section eq 'pending') {
2410                 $conf->{delete} = $value; # we parse this later
2411             } else {
2412                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2413             }
2414         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2415             my $key = $1;
2416             my $value = $2;
2417             eval { $value = check_type($key, $value); };
2418             if ($@) {
2419                 warn "vm $vmid - unable to parse value of '$key' - $@";
2420             } else {
2421                 $key = 'ide2' if $key eq 'cdrom';
2422                 my $fmt = $confdesc->{$key}->{format};
2423                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2424                     my $v = parse_drive($key, $value);
2425                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2426                         $v->{file} = $volid;
2427                         $value = print_drive($vmid, $v);
2428                     } else {
2429                         warn "vm $vmid - unable to parse value of '$key'\n";
2430                         next;
2431                     }
2432                 }
2433
2434                 $conf->{$key} = $value;
2435             }
2436         }
2437     }
2438
2439     if (defined($descr)) {
2440         $descr =~ s/\s+$//;
2441         $conf->{description} = $descr;
2442     }
2443     delete $res->{snapstate}; # just to be sure
2444
2445     return $res;
2446 }
2447
2448 sub write_vm_config {
2449     my ($filename, $conf) = @_;
2450
2451     delete $conf->{snapstate}; # just to be sure
2452
2453     if ($conf->{cdrom}) {
2454         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2455         $conf->{ide2} = $conf->{cdrom};
2456         delete $conf->{cdrom};
2457     }
2458
2459     # we do not use 'smp' any longer
2460     if ($conf->{sockets}) {
2461         delete $conf->{smp};
2462     } elsif ($conf->{smp}) {
2463         $conf->{sockets} = $conf->{smp};
2464         delete $conf->{cores};
2465         delete $conf->{smp};
2466     }
2467
2468     my $used_volids = {};
2469
2470     my $cleanup_config = sub {
2471         my ($cref, $pending, $snapname) = @_;
2472
2473         foreach my $key (keys %$cref) {
2474             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2475                 $key eq 'snapstate' || $key eq 'pending';
2476             my $value = $cref->{$key};
2477             if ($key eq 'delete') {
2478                 die "propertry 'delete' is only allowed in [PENDING]\n"
2479                     if !$pending;
2480                 # fixme: check syntax?
2481                 next;
2482             }
2483             eval { $value = check_type($key, $value); };
2484             die "unable to parse value of '$key' - $@" if $@;
2485
2486             $cref->{$key} = $value;
2487
2488             if (!$snapname && is_valid_drivename($key)) {
2489                 my $drive = parse_drive($key, $value);
2490                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2491             }
2492         }
2493     };
2494
2495     &$cleanup_config($conf);
2496
2497     &$cleanup_config($conf->{pending}, 1);
2498
2499     foreach my $snapname (keys %{$conf->{snapshots}}) {
2500         die "internal error" if $snapname eq 'pending';
2501         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2502     }
2503
2504     # remove 'unusedX' settings if we re-add a volume
2505     foreach my $key (keys %$conf) {
2506         my $value = $conf->{$key};
2507         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2508             delete $conf->{$key};
2509         }
2510     }
2511
2512     my $generate_raw_config = sub {
2513         my ($conf, $pending) = @_;
2514
2515         my $raw = '';
2516
2517         # add description as comment to top of file
2518         if (defined(my $descr = $conf->{description})) {
2519             if ($descr) {
2520                 foreach my $cl (split(/\n/, $descr)) {
2521                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2522                 }
2523             } else {
2524                 $raw .= "#\n" if $pending;
2525             }
2526         }
2527
2528         foreach my $key (sort keys %$conf) {
2529             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2530             $raw .= "$key: $conf->{$key}\n";
2531         }
2532         return $raw;
2533     };
2534
2535     my $raw = &$generate_raw_config($conf);
2536
2537     if (scalar(keys %{$conf->{pending}})){
2538         $raw .= "\n[PENDING]\n";
2539         $raw .= &$generate_raw_config($conf->{pending}, 1);
2540     }
2541
2542     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2543         $raw .= "\n[$snapname]\n";
2544         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2545     }
2546
2547     return $raw;
2548 }
2549
2550 sub load_defaults {
2551
2552     my $res = {};
2553
2554     # we use static defaults from our JSON schema configuration
2555     foreach my $key (keys %$confdesc) {
2556         if (defined(my $default = $confdesc->{$key}->{default})) {
2557             $res->{$key} = $default;
2558         }
2559     }
2560
2561     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2562     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2563
2564     return $res;
2565 }
2566
2567 sub config_list {
2568     my $vmlist = PVE::Cluster::get_vmlist();
2569     my $res = {};
2570     return $res if !$vmlist || !$vmlist->{ids};
2571     my $ids = $vmlist->{ids};
2572
2573     foreach my $vmid (keys %$ids) {
2574         my $d = $ids->{$vmid};
2575         next if !$d->{node} || $d->{node} ne $nodename;
2576         next if !$d->{type} || $d->{type} ne 'qemu';
2577         $res->{$vmid}->{exists} = 1;
2578     }
2579     return $res;
2580 }
2581
2582 # test if VM uses local resources (to prevent migration)
2583 sub check_local_resources {
2584     my ($conf, $noerr) = @_;
2585
2586     my $loc_res = 0;
2587
2588     $loc_res = 1 if $conf->{hostusb}; # old syntax
2589     $loc_res = 1 if $conf->{hostpci}; # old syntax
2590
2591     foreach my $k (keys %$conf) {
2592         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2593         # sockets are safe: they will recreated be on the target side post-migrate
2594         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2595         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2596     }
2597
2598     die "VM uses local resources\n" if $loc_res && !$noerr;
2599
2600     return $loc_res;
2601 }
2602
2603 # check if used storages are available on all nodes (use by migrate)
2604 sub check_storage_availability {
2605     my ($storecfg, $conf, $node) = @_;
2606
2607     foreach_drive($conf, sub {
2608         my ($ds, $drive) = @_;
2609
2610         my $volid = $drive->{file};
2611         return if !$volid;
2612
2613         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2614         return if !$sid;
2615
2616         # check if storage is available on both nodes
2617         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2618         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2619    });
2620 }
2621
2622 # list nodes where all VM images are available (used by has_feature API)
2623 sub shared_nodes {
2624     my ($conf, $storecfg) = @_;
2625
2626     my $nodelist = PVE::Cluster::get_nodelist();
2627     my $nodehash = { map { $_ => 1 } @$nodelist };
2628     my $nodename = PVE::INotify::nodename();
2629
2630     foreach_drive($conf, sub {
2631         my ($ds, $drive) = @_;
2632
2633         my $volid = $drive->{file};
2634         return if !$volid;
2635
2636         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2637         if ($storeid) {
2638             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2639             if ($scfg->{disable}) {
2640                 $nodehash = {};
2641             } elsif (my $avail = $scfg->{nodes}) {
2642                 foreach my $node (keys %$nodehash) {
2643                     delete $nodehash->{$node} if !$avail->{$node};
2644                 }
2645             } elsif (!$scfg->{shared}) {
2646                 foreach my $node (keys %$nodehash) {
2647                     delete $nodehash->{$node} if $node ne $nodename
2648                 }
2649             }
2650         }
2651     });
2652
2653     return $nodehash
2654 }
2655
2656 sub check_cmdline {
2657     my ($pidfile, $pid) = @_;
2658
2659     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2660     if (defined($fh)) {
2661         my $line = <$fh>;
2662         $fh->close;
2663         return undef if !$line;
2664         my @param = split(/\0/, $line);
2665
2666         my $cmd = $param[0];
2667         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2668
2669         for (my $i = 0; $i < scalar (@param); $i++) {
2670             my $p = $param[$i];
2671             next if !$p;
2672             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2673                 my $p = $param[$i+1];
2674                 return 1 if $p && ($p eq $pidfile);
2675                 return undef;
2676             }
2677         }
2678     }
2679     return undef;
2680 }
2681
2682 sub check_running {
2683     my ($vmid, $nocheck, $node) = @_;
2684
2685     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2686
2687     die "unable to find configuration file for VM $vmid - no such machine\n"
2688         if !$nocheck && ! -f $filename;
2689
2690     my $pidfile = pidfile_name($vmid);
2691
2692     if (my $fd = IO::File->new("<$pidfile")) {
2693         my $st = stat($fd);
2694         my $line = <$fd>;
2695         close($fd);
2696
2697         my $mtime = $st->mtime;
2698         if ($mtime > time()) {
2699             warn "file '$filename' modified in future\n";
2700         }
2701
2702         if ($line =~ m/^(\d+)$/) {
2703             my $pid = $1;
2704             if (check_cmdline($pidfile, $pid)) {
2705                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2706                     return $pid;
2707                 }
2708             }
2709         }
2710     }
2711
2712     return undef;
2713 }
2714
2715 sub vzlist {
2716
2717     my $vzlist = config_list();
2718
2719     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2720
2721     while (defined(my $de = $fd->read)) {
2722         next if $de !~ m/^(\d+)\.pid$/;
2723         my $vmid = $1;
2724         next if !defined($vzlist->{$vmid});
2725         if (my $pid = check_running($vmid)) {
2726             $vzlist->{$vmid}->{pid} = $pid;
2727         }
2728     }
2729
2730     return $vzlist;
2731 }
2732
2733 sub disksize {
2734     my ($storecfg, $conf) = @_;
2735
2736     my $bootdisk = $conf->{bootdisk};
2737     return undef if !$bootdisk;
2738     return undef if !is_valid_drivename($bootdisk);
2739
2740     return undef if !$conf->{$bootdisk};
2741
2742     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2743     return undef if !defined($drive);
2744
2745     return undef if drive_is_cdrom($drive);
2746
2747     my $volid = $drive->{file};
2748     return undef if !$volid;
2749
2750     return $drive->{size};
2751 }
2752
2753 my $last_proc_pid_stat;
2754
2755 # get VM status information
2756 # This must be fast and should not block ($full == false)
2757 # We only query KVM using QMP if $full == true (this can be slow)
2758 sub vmstatus {
2759     my ($opt_vmid, $full) = @_;
2760
2761     my $res = {};
2762
2763     my $storecfg = PVE::Storage::config();
2764
2765     my $list = vzlist();
2766     my $defaults = load_defaults();
2767
2768     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2769
2770     my $cpucount = $cpuinfo->{cpus} || 1;
2771
2772     foreach my $vmid (keys %$list) {
2773         next if $opt_vmid && ($vmid ne $opt_vmid);
2774
2775         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2776         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2777
2778         my $d = {};
2779         $d->{pid} = $list->{$vmid}->{pid};
2780
2781         # fixme: better status?
2782         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2783
2784         my $size = disksize($storecfg, $conf);
2785         if (defined($size)) {
2786             $d->{disk} = 0; # no info available
2787             $d->{maxdisk} = $size;
2788         } else {
2789             $d->{disk} = 0;
2790             $d->{maxdisk} = 0;
2791         }
2792
2793         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2794             * ($conf->{cores} || $defaults->{cores});
2795         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2796         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2797
2798         $d->{name} = $conf->{name} || "VM $vmid";
2799         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2800             : $defaults->{memory}*(1024*1024);
2801
2802         if ($conf->{balloon}) {
2803             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2804             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2805                 : $defaults->{shares};
2806         }
2807
2808         $d->{uptime} = 0;
2809         $d->{cpu} = 0;
2810         $d->{mem} = 0;
2811
2812         $d->{netout} = 0;
2813         $d->{netin} = 0;
2814
2815         $d->{diskread} = 0;
2816         $d->{diskwrite} = 0;
2817
2818         $d->{template} = PVE::QemuConfig->is_template($conf);
2819
2820         $d->{serial} = 1 if conf_has_serial($conf);
2821
2822         $res->{$vmid} = $d;
2823     }
2824
2825     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2826     foreach my $dev (keys %$netdev) {
2827         next if $dev !~ m/^tap([1-9]\d*)i/;
2828         my $vmid = $1;
2829         my $d = $res->{$vmid};
2830         next if !$d;
2831
2832         $d->{netout} += $netdev->{$dev}->{receive};
2833         $d->{netin} += $netdev->{$dev}->{transmit};
2834
2835         if ($full) {
2836             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2837             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2838         }
2839
2840     }
2841
2842     my $ctime = gettimeofday;
2843
2844     foreach my $vmid (keys %$list) {
2845
2846         my $d = $res->{$vmid};
2847         my $pid = $d->{pid};
2848         next if !$pid;
2849
2850         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2851         next if !$pstat; # not running
2852
2853         my $used = $pstat->{utime} + $pstat->{stime};
2854
2855         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2856
2857         if ($pstat->{vsize}) {
2858             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2859         }
2860
2861         my $old = $last_proc_pid_stat->{$pid};
2862         if (!$old) {
2863             $last_proc_pid_stat->{$pid} = {
2864                 time => $ctime,
2865                 used => $used,
2866                 cpu => 0,
2867             };
2868             next;
2869         }
2870
2871         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2872
2873         if ($dtime > 1000) {
2874             my $dutime = $used -  $old->{used};
2875
2876             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2877             $last_proc_pid_stat->{$pid} = {
2878                 time => $ctime,
2879                 used => $used,
2880                 cpu => $d->{cpu},
2881             };
2882         } else {
2883             $d->{cpu} = $old->{cpu};
2884         }
2885     }
2886
2887     return $res if !$full;
2888
2889     my $qmpclient = PVE::QMPClient->new();
2890
2891     my $ballooncb = sub {
2892         my ($vmid, $resp) = @_;
2893
2894         my $info = $resp->{'return'};
2895         return if !$info->{max_mem};
2896
2897         my $d = $res->{$vmid};
2898
2899         # use memory assigned to VM
2900         $d->{maxmem} = $info->{max_mem};
2901         $d->{balloon} = $info->{actual};
2902
2903         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2904             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2905             $d->{freemem} = $info->{free_mem};
2906         }
2907
2908         $d->{ballooninfo} = $info;
2909     };
2910
2911     my $blockstatscb = sub {
2912         my ($vmid, $resp) = @_;
2913         my $data = $resp->{'return'} || [];
2914         my $totalrdbytes = 0;
2915         my $totalwrbytes = 0;
2916
2917         for my $blockstat (@$data) {
2918             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2919             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2920
2921             $blockstat->{device} =~ s/drive-//;
2922             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2923         }
2924         $res->{$vmid}->{diskread} = $totalrdbytes;
2925         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2926     };
2927
2928     my $statuscb = sub {
2929         my ($vmid, $resp) = @_;
2930
2931         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2932         # this fails if ballon driver is not loaded, so this must be
2933         # the last commnand (following command are aborted if this fails).
2934         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2935
2936         my $status = 'unknown';
2937         if (!defined($status = $resp->{'return'}->{status})) {
2938             warn "unable to get VM status\n";
2939             return;
2940         }
2941
2942         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2943     };
2944
2945     foreach my $vmid (keys %$list) {
2946         next if $opt_vmid && ($vmid ne $opt_vmid);
2947         next if !$res->{$vmid}->{pid}; # not running
2948         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2949     }
2950
2951     $qmpclient->queue_execute(undef, 2);
2952
2953     foreach my $vmid (keys %$list) {
2954         next if $opt_vmid && ($vmid ne $opt_vmid);
2955         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2956     }
2957
2958     return $res;
2959 }
2960
2961 sub foreach_drive {
2962     my ($conf, $func, @param) = @_;
2963
2964     foreach my $ds (valid_drive_names()) {
2965         next if !defined($conf->{$ds});
2966
2967         my $drive = parse_drive($ds, $conf->{$ds});
2968         next if !$drive;
2969
2970         &$func($ds, $drive, @param);
2971     }
2972 }
2973
2974 sub foreach_volid {
2975     my ($conf, $func, @param) = @_;
2976
2977     my $volhash = {};
2978
2979     my $test_volid = sub {
2980         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2981
2982         return if !$volid;
2983
2984         $volhash->{$volid}->{cdrom} //= 1;
2985         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2986
2987         $volhash->{$volid}->{replicate} //= 0;
2988         $volhash->{$volid}->{replicate} = 1 if $replicate;
2989
2990         $volhash->{$volid}->{shared} //= 0;
2991         $volhash->{$volid}->{shared} = 1 if $shared;
2992
2993         $volhash->{$volid}->{referenced_in_config} //= 0;
2994         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2995
2996         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2997             if defined($snapname);
2998     };
2999
3000     foreach_drive($conf, sub {
3001         my ($ds, $drive) = @_;
3002         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3003     });
3004
3005     foreach my $snapname (keys %{$conf->{snapshots}}) {
3006         my $snap = $conf->{snapshots}->{$snapname};
3007         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3008         foreach_drive($snap, sub {
3009             my ($ds, $drive) = @_;
3010             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3011         });
3012     }
3013
3014     foreach my $volid (keys %$volhash) {
3015         &$func($volid, $volhash->{$volid}, @param);
3016     }
3017 }
3018
3019 sub conf_has_serial {
3020     my ($conf) = @_;
3021
3022     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3023         if ($conf->{"serial$i"}) {
3024             return 1;
3025         }
3026     }
3027
3028     return 0;
3029 }
3030
3031 sub vga_conf_has_spice {
3032     my ($vga) = @_;
3033
3034     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3035
3036     return $1 || 1;
3037 }
3038
3039 sub config_to_command {
3040     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3041
3042     my $cmd = [];
3043     my $globalFlags = [];
3044     my $machineFlags = [];
3045     my $rtcFlags = [];
3046     my $cpuFlags = [];
3047     my $devices = [];
3048     my $pciaddr = '';
3049     my $bridges = {};
3050     my $kvmver = kvm_user_version();
3051     my $vernum = 0; # unknown
3052     my $ostype = $conf->{ostype};
3053     my $winversion = windows_version($ostype);
3054     my $kvm = $conf->{kvm} // 1;
3055
3056     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3057
3058     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3059         $vernum = $1*1000000+$2*1000;
3060     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3061         $vernum = $1*1000000+$2*1000+$3;
3062     }
3063
3064     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3065
3066     my $have_ovz = -f '/proc/vz/vestat';
3067
3068     my $q35 = machine_type_is_q35($conf);
3069     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3070     my $machine_type = $forcemachine || $conf->{machine};
3071     my $use_old_bios_files = undef;
3072     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3073
3074     my $cpuunits = defined($conf->{cpuunits}) ?
3075             $conf->{cpuunits} : $defaults->{cpuunits};
3076
3077     push @$cmd, '/usr/bin/kvm';
3078
3079     push @$cmd, '-id', $vmid;
3080
3081     my $use_virtio = 0;
3082
3083     my $qmpsocket = qmp_socket($vmid);
3084     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3085     push @$cmd, '-mon', "chardev=qmp,mode=control";
3086
3087
3088     push @$cmd, '-pidfile' , pidfile_name($vmid);
3089
3090     push @$cmd, '-daemonize';
3091
3092     if ($conf->{smbios1}) {
3093         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3094     }
3095
3096     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3097         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3098
3099         my $path;
3100         my $format;
3101         if (my $efidisk = $conf->{efidisk0}) {
3102             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3103             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3104             $format = $d->{format};
3105             if ($storeid) {
3106                 $path = PVE::Storage::path($storecfg, $d->{file});
3107                 if (!defined($format)) {
3108                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3109                     $format = qemu_img_format($scfg, $volname);
3110                 }
3111             } else {
3112                 $path = $d->{file};
3113                 die "efidisk format must be specified\n"
3114                     if !defined($format);
3115             }
3116         } else {
3117             warn "no efidisk configured! Using temporary efivars disk.\n";
3118             $path = "/tmp/$vmid-ovmf.fd";
3119             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3120             $format = 'raw';
3121         }
3122
3123         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3124         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3125     }
3126
3127
3128     # add usb controllers
3129     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3130     push @$devices, @usbcontrollers if @usbcontrollers;
3131     my $vga = $conf->{vga};
3132
3133     my $qxlnum = vga_conf_has_spice($vga);
3134     $vga = 'qxl' if $qxlnum;
3135
3136     if (!$vga) {
3137         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3138             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3139         } else {
3140             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3141         }
3142     }
3143
3144     # enable absolute mouse coordinates (needed by vnc)
3145     my $tablet;
3146     if (defined($conf->{tablet})) {
3147         $tablet = $conf->{tablet};
3148     } else {
3149         $tablet = $defaults->{tablet};
3150         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3151         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3152     }
3153
3154     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3155
3156     my $kvm_off = 0;
3157     my $gpu_passthrough;
3158
3159     # host pci devices
3160     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3161         my $d = parse_hostpci($conf->{"hostpci$i"});
3162         next if !$d;
3163
3164         my $pcie = $d->{pcie};
3165         if($pcie){
3166             die "q35 machine model is not enabled" if !$q35;
3167             $pciaddr = print_pcie_addr("hostpci$i");
3168         }else{
3169             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3170         }
3171
3172         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3173         my $romfile = $d->{romfile};
3174
3175         my $xvga = '';
3176         if ($d->{'x-vga'}) {
3177             $xvga = ',x-vga=on';
3178             $kvm_off = 1;
3179             $vga = 'none';
3180             $gpu_passthrough = 1;
3181
3182             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3183                 $xvga = "";
3184             }
3185         }
3186         my $pcidevices = $d->{pciid};
3187         my $multifunction = 1 if @$pcidevices > 1;
3188
3189         my $j=0;
3190         foreach my $pcidevice (@$pcidevices) {
3191
3192             my $id = "hostpci$i";
3193             $id .= ".$j" if $multifunction;
3194             my $addr = $pciaddr;
3195             $addr .= ".$j" if $multifunction;
3196             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3197
3198             if($j == 0){
3199                 $devicestr .= "$rombar$xvga";
3200                 $devicestr .= ",multifunction=on" if $multifunction;
3201                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3202             }
3203
3204             push @$devices, '-device', $devicestr;
3205             $j++;
3206         }
3207     }
3208
3209     # usb devices
3210     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3211     push @$devices, @usbdevices if @usbdevices;
3212     # serial devices
3213     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3214         if (my $path = $conf->{"serial$i"}) {
3215             if ($path eq 'socket') {
3216                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3217                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3218                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3219             } else {
3220                 die "no such serial device\n" if ! -c $path;
3221                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3222                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3223             }
3224         }
3225     }
3226
3227     # parallel devices
3228     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3229         if (my $path = $conf->{"parallel$i"}) {
3230             die "no such parallel device\n" if ! -c $path;
3231             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3232             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3233             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3234         }
3235     }
3236
3237     my $vmname = $conf->{name} || "vm$vmid";
3238
3239     push @$cmd, '-name', $vmname;
3240
3241     my $sockets = 1;
3242     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3243     $sockets = $conf->{sockets} if  $conf->{sockets};
3244
3245     my $cores = $conf->{cores} || 1;
3246
3247     my $maxcpus = $sockets * $cores;
3248
3249     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3250
3251     my $allowed_vcpus = $cpuinfo->{cpus};
3252
3253     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3254         if ($allowed_vcpus < $maxcpus);
3255
3256     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3257
3258         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3259         for (my $i = 2; $i <= $vcpus; $i++)  {
3260             my $cpustr = print_cpu_device($conf,$i);
3261             push @$cmd, '-device', $cpustr;
3262         }
3263
3264     } else {
3265
3266         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3267     }
3268     push @$cmd, '-nodefaults';
3269
3270     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3271
3272     my $bootindex_hash = {};
3273     my $i = 1;
3274     foreach my $o (split(//, $bootorder)) {
3275         $bootindex_hash->{$o} = $i*100;
3276         $i++;
3277     }
3278
3279     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3280
3281     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3282
3283     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3284
3285     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3286
3287     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3288         my $socket = vnc_socket($vmid);
3289         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3290     } else {
3291         push @$cmd, '-nographic';
3292     }
3293
3294     # time drift fix
3295     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3296
3297     my $useLocaltime = $conf->{localtime};
3298
3299     if ($winversion >= 5) { # windows
3300         $useLocaltime = 1 if !defined($conf->{localtime});
3301
3302         # use time drift fix when acpi is enabled
3303         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3304             $tdf = 1 if !defined($conf->{tdf});
3305         }
3306     }
3307
3308     if ($winversion >= 6) {
3309         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3310         push @$cmd, '-no-hpet';
3311     }
3312
3313     push @$rtcFlags, 'driftfix=slew' if $tdf;
3314
3315     if (!$kvm) {
3316         push @$machineFlags, 'accel=tcg';
3317     }
3318
3319     if ($machine_type) {
3320         push @$machineFlags, "type=${machine_type}";
3321     }
3322
3323     if ($conf->{startdate}) {
3324         push @$rtcFlags, "base=$conf->{startdate}";
3325     } elsif ($useLocaltime) {
3326         push @$rtcFlags, 'base=localtime';
3327     }
3328
3329     my $cpu = $kvm ? "kvm64" : "qemu64";
3330     if (my $cputype = $conf->{cpu}) {
3331         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3332             or die "Cannot parse cpu description: $cputype\n";
3333         $cpu = $cpuconf->{cputype};
3334         $kvm_off = 1 if $cpuconf->{hidden};
3335
3336         if (defined(my $flags = $cpuconf->{flags})) {
3337             push @$cpuFlags, split(";", $flags);
3338         }
3339     }
3340
3341     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3342
3343     push @$cpuFlags , '-x2apic'
3344         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3345
3346     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3347
3348     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3349
3350     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3351
3352         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3353         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3354     }
3355
3356     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3357
3358     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3359
3360     push @$cpuFlags, 'kvm=off' if $kvm_off;
3361
3362     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3363         die "internal error"; # should not happen
3364
3365     push @$cpuFlags, "vendor=${cpu_vendor}"
3366         if $cpu_vendor ne 'default';
3367
3368     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3369
3370     push @$cmd, '-cpu', $cpu;
3371
3372     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3373
3374     push @$cmd, '-S' if $conf->{freeze};
3375
3376     # set keyboard layout
3377     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3378     push @$cmd, '-k', $kb if $kb;
3379
3380     # enable sound
3381     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3382     #push @$cmd, '-soundhw', 'es1370';
3383     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3384
3385     if($conf->{agent}) {
3386         my $qgasocket = qmp_socket($vmid, 1);
3387         my $pciaddr = print_pci_addr("qga0", $bridges);
3388         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3389         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3390         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3391     }
3392
3393     my $spice_port;
3394
3395     if ($qxlnum) {
3396         if ($qxlnum > 1) {
3397             if ($winversion){
3398                 for(my $i = 1; $i < $qxlnum; $i++){
3399                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3400                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3401                 }
3402             } else {
3403                 # assume other OS works like Linux
3404                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3405                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3406             }
3407         }
3408
3409         my $pciaddr = print_pci_addr("spice", $bridges);
3410
3411         my $nodename = PVE::INotify::nodename();
3412         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3413         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3414         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3415         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3416         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3417
3418         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3419
3420         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3421         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3422         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3423     }
3424
3425     # enable balloon by default, unless explicitly disabled
3426     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3427         $pciaddr = print_pci_addr("balloon0", $bridges);
3428         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3429     }
3430
3431     if ($conf->{watchdog}) {
3432         my $wdopts = parse_watchdog($conf->{watchdog});
3433         $pciaddr = print_pci_addr("watchdog", $bridges);
3434         my $watchdog = $wdopts->{model} || 'i6300esb';
3435         push @$devices, '-device', "$watchdog$pciaddr";
3436         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3437     }
3438
3439     my $vollist = [];
3440     my $scsicontroller = {};
3441     my $ahcicontroller = {};
3442     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3443
3444     # Add iscsi initiator name if available
3445     if (my $initiator = get_initiator_name()) {
3446         push @$devices, '-iscsi', "initiator-name=$initiator";
3447     }
3448
3449     foreach_drive($conf, sub {
3450         my ($ds, $drive) = @_;
3451
3452         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3453             push @$vollist, $drive->{file};
3454         }
3455
3456         # ignore efidisk here, already added in bios/fw handling code above
3457         return if $drive->{interface} eq 'efidisk';
3458
3459         $use_virtio = 1 if $ds =~ m/^virtio/;
3460
3461         if (drive_is_cdrom ($drive)) {
3462             if ($bootindex_hash->{d}) {
3463                 $drive->{bootindex} = $bootindex_hash->{d};
3464                 $bootindex_hash->{d} += 1;
3465             }
3466         } else {
3467             if ($bootindex_hash->{c}) {
3468                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3469                 $bootindex_hash->{c} += 1;
3470             }
3471         }
3472
3473         if($drive->{interface} eq 'virtio'){
3474            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3475         }
3476
3477         if ($drive->{interface} eq 'scsi') {
3478
3479             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3480
3481             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3482             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3483
3484             my $iothread = '';
3485             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3486                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3487                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3488             } elsif ($drive->{iothread}) {
3489                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3490             }
3491
3492             my $queues = '';
3493             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3494                 $queues = ",num_queues=$drive->{queues}";
3495             }
3496
3497             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3498             $scsicontroller->{$controller}=1;
3499         }
3500
3501         if ($drive->{interface} eq 'sata') {
3502            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3503            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3504            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3505            $ahcicontroller->{$controller}=1;
3506         }
3507
3508         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3509         push @$devices, '-drive',$drive_cmd;
3510         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3511     });
3512
3513     for (my $i = 0; $i < $MAX_NETS; $i++) {
3514          next if !$conf->{"net$i"};
3515          my $d = parse_net($conf->{"net$i"});
3516          next if !$d;
3517
3518          $use_virtio = 1 if $d->{model} eq 'virtio';
3519
3520          if ($bootindex_hash->{n}) {
3521             $d->{bootindex} = $bootindex_hash->{n};
3522             $bootindex_hash->{n} += 1;
3523          }
3524
3525          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3526          push @$devices, '-netdev', $netdevfull;
3527
3528          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3529          push @$devices, '-device', $netdevicefull;
3530     }
3531
3532     if (!$q35) {
3533         # add pci bridges
3534         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3535            $bridges->{1} = 1;
3536            $bridges->{2} = 1;
3537         }
3538
3539         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3540
3541         while (my ($k, $v) = each %$bridges) {
3542             $pciaddr = print_pci_addr("pci.$k");
3543             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3544         }
3545     }
3546
3547     # add custom args
3548     if ($conf->{args}) {
3549         my $aa = PVE::Tools::split_args($conf->{args});
3550         push @$cmd, @$aa;
3551     }
3552
3553     push @$cmd, @$devices;
3554     push @$cmd, '-rtc', join(',', @$rtcFlags)
3555         if scalar(@$rtcFlags);
3556     push @$cmd, '-machine', join(',', @$machineFlags)
3557         if scalar(@$machineFlags);
3558     push @$cmd, '-global', join(',', @$globalFlags)
3559         if scalar(@$globalFlags);
3560
3561     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3562 }
3563
3564 sub vnc_socket {
3565     my ($vmid) = @_;
3566     return "${var_run_tmpdir}/$vmid.vnc";
3567 }
3568
3569 sub spice_port {
3570     my ($vmid) = @_;
3571
3572     my $res = vm_mon_cmd($vmid, 'query-spice');
3573
3574     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3575 }
3576
3577 sub qmp_socket {
3578     my ($vmid, $qga) = @_;
3579     my $sockettype = $qga ? 'qga' : 'qmp';
3580     return "${var_run_tmpdir}/$vmid.$sockettype";
3581 }
3582
3583 sub pidfile_name {
3584     my ($vmid) = @_;
3585     return "${var_run_tmpdir}/$vmid.pid";
3586 }
3587
3588 sub vm_devices_list {
3589     my ($vmid) = @_;
3590
3591     my $res = vm_mon_cmd($vmid, 'query-pci');
3592     my $devices = {};
3593     foreach my $pcibus (@$res) {
3594         foreach my $device (@{$pcibus->{devices}}) {
3595             next if !$device->{'qdev_id'};
3596             if ($device->{'pci_bridge'}) {
3597                 $devices->{$device->{'qdev_id'}} = 1;
3598                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3599                     next if !$bridge_device->{'qdev_id'};
3600                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3601                     $devices->{$device->{'qdev_id'}}++;
3602                 }
3603             } else {
3604                 $devices->{$device->{'qdev_id'}} = 1;
3605             }
3606         }
3607     }
3608
3609     my $resblock = vm_mon_cmd($vmid, 'query-block');
3610     foreach my $block (@$resblock) {
3611         if($block->{device} =~ m/^drive-(\S+)/){
3612                 $devices->{$1} = 1;
3613         }
3614     }
3615
3616     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3617     foreach my $mice (@$resmice) {
3618         if ($mice->{name} eq 'QEMU HID Tablet') {
3619             $devices->{tablet} = 1;
3620             last;
3621         }
3622     }
3623
3624     # for usb devices there is no query-usb
3625     # but we can iterate over the entries in
3626     # qom-list path=/machine/peripheral
3627     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3628     foreach my $per (@$resperipheral) {
3629         if ($per->{name} =~ m/^usb\d+$/) {
3630             $devices->{$per->{name}} = 1;
3631         }
3632     }
3633
3634     return $devices;
3635 }
3636
3637 sub vm_deviceplug {
3638     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3639
3640     my $q35 = machine_type_is_q35($conf);
3641
3642     my $devices_list = vm_devices_list($vmid);
3643     return 1 if defined($devices_list->{$deviceid});
3644
3645     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3646
3647     if ($deviceid eq 'tablet') {
3648
3649         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3650
3651     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3652
3653         die "usb hotplug currently not reliable\n";
3654         # since we can't reliably hot unplug all added usb devices
3655         # and usb passthrough disables live migration
3656         # we disable usb hotplugging for now
3657         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3658
3659     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3660
3661         qemu_iothread_add($vmid, $deviceid, $device);
3662
3663         qemu_driveadd($storecfg, $vmid, $device);
3664         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3665
3666         qemu_deviceadd($vmid, $devicefull);
3667         eval { qemu_deviceaddverify($vmid, $deviceid); };
3668         if (my $err = $@) {
3669             eval { qemu_drivedel($vmid, $deviceid); };
3670             warn $@ if $@;
3671             die $err;
3672         }
3673
3674     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3675
3676
3677         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3678         my $pciaddr = print_pci_addr($deviceid);
3679         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3680
3681         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3682
3683         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3684             qemu_iothread_add($vmid, $deviceid, $device);
3685             $devicefull .= ",iothread=iothread-$deviceid";
3686         }
3687
3688         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3689             $devicefull .= ",num_queues=$device->{queues}";
3690         }
3691
3692         qemu_deviceadd($vmid, $devicefull);
3693         qemu_deviceaddverify($vmid, $deviceid);
3694
3695     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3696
3697         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3698         qemu_driveadd($storecfg, $vmid, $device);
3699
3700         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3701         eval { qemu_deviceadd($vmid, $devicefull); };
3702         if (my $err = $@) {
3703             eval { qemu_drivedel($vmid, $deviceid); };
3704             warn $@ if $@;
3705             die $err;
3706         }
3707
3708     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3709
3710         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3711
3712         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3713         my $use_old_bios_files = undef;
3714         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3715
3716         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3717         qemu_deviceadd($vmid, $netdevicefull);
3718         eval { qemu_deviceaddverify($vmid, $deviceid); };
3719         if (my $err = $@) {
3720             eval { qemu_netdevdel($vmid, $deviceid); };
3721             warn $@ if $@;
3722             die $err;
3723         }
3724
3725     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3726
3727         my $bridgeid = $2;
3728         my $pciaddr = print_pci_addr($deviceid);
3729         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3730
3731         qemu_deviceadd($vmid, $devicefull);
3732         qemu_deviceaddverify($vmid, $deviceid);
3733
3734     } else {
3735         die "can't hotplug device '$deviceid'\n";
3736     }
3737
3738     return 1;
3739 }
3740
3741 # fixme: this should raise exceptions on error!
3742 sub vm_deviceunplug {
3743     my ($vmid, $conf, $deviceid) = @_;
3744
3745     my $devices_list = vm_devices_list($vmid);
3746     return 1 if !defined($devices_list->{$deviceid});
3747
3748     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3749
3750     if ($deviceid eq 'tablet') {
3751
3752         qemu_devicedel($vmid, $deviceid);
3753
3754     } elsif ($deviceid =~ m/^usb\d+$/) {
3755
3756         die "usb hotplug currently not reliable\n";
3757         # when unplugging usb devices this way,
3758         # there may be remaining usb controllers/hubs
3759         # so we disable it for now
3760         qemu_devicedel($vmid, $deviceid);
3761         qemu_devicedelverify($vmid, $deviceid);
3762
3763     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3764
3765         qemu_devicedel($vmid, $deviceid);
3766         qemu_devicedelverify($vmid, $deviceid);
3767         qemu_drivedel($vmid, $deviceid);
3768         qemu_iothread_del($conf, $vmid, $deviceid);
3769
3770     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3771
3772         qemu_devicedel($vmid, $deviceid);
3773         qemu_devicedelverify($vmid, $deviceid);
3774         qemu_iothread_del($conf, $vmid, $deviceid);
3775
3776     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3777
3778         qemu_devicedel($vmid, $deviceid);
3779         qemu_drivedel($vmid, $deviceid);
3780         qemu_deletescsihw($conf, $vmid, $deviceid);
3781
3782     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3783
3784         qemu_devicedel($vmid, $deviceid);
3785         qemu_devicedelverify($vmid, $deviceid);
3786         qemu_netdevdel($vmid, $deviceid);
3787
3788     } else {
3789         die "can't unplug device '$deviceid'\n";
3790     }
3791
3792     return 1;
3793 }
3794
3795 sub qemu_deviceadd {
3796     my ($vmid, $devicefull) = @_;
3797
3798     $devicefull = "driver=".$devicefull;
3799     my %options =  split(/[=,]/, $devicefull);
3800
3801     vm_mon_cmd($vmid, "device_add" , %options);
3802 }
3803
3804 sub qemu_devicedel {
3805     my ($vmid, $deviceid) = @_;
3806
3807     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3808 }
3809
3810 sub qemu_iothread_add {
3811     my($vmid, $deviceid, $device) = @_;
3812
3813     if ($device->{iothread}) {
3814         my $iothreads = vm_iothreads_list($vmid);
3815         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3816     }
3817 }
3818
3819 sub qemu_iothread_del {
3820     my($conf, $vmid, $deviceid) = @_;
3821
3822     my $device = parse_drive($deviceid, $conf->{$deviceid});
3823     if ($device->{iothread}) {
3824         my $iothreads = vm_iothreads_list($vmid);
3825         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3826     }
3827 }
3828
3829 sub qemu_objectadd {
3830     my($vmid, $objectid, $qomtype) = @_;
3831
3832     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3833
3834     return 1;
3835 }
3836
3837 sub qemu_objectdel {
3838     my($vmid, $objectid) = @_;
3839
3840     vm_mon_cmd($vmid, "object-del", id => $objectid);
3841
3842     return 1;
3843 }
3844
3845 sub qemu_driveadd {
3846     my ($storecfg, $vmid, $device) = @_;
3847
3848     my $drive = print_drive_full($storecfg, $vmid, $device);
3849     $drive =~ s/\\/\\\\/g;
3850     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3851
3852     # If the command succeeds qemu prints: "OK"
3853     return 1 if $ret =~ m/OK/s;
3854
3855     die "adding drive failed: $ret\n";
3856 }
3857
3858 sub qemu_drivedel {
3859     my($vmid, $deviceid) = @_;
3860
3861     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3862     $ret =~ s/^\s+//;
3863
3864     return 1 if $ret eq "";
3865
3866     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3867     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3868
3869     die "deleting drive $deviceid failed : $ret\n";
3870 }
3871
3872 sub qemu_deviceaddverify {
3873     my ($vmid, $deviceid) = @_;
3874
3875     for (my $i = 0; $i <= 5; $i++) {
3876          my $devices_list = vm_devices_list($vmid);
3877          return 1 if defined($devices_list->{$deviceid});
3878          sleep 1;
3879     }
3880
3881     die "error on hotplug device '$deviceid'\n";
3882 }
3883
3884
3885 sub qemu_devicedelverify {
3886     my ($vmid, $deviceid) = @_;
3887
3888     # need to verify that the device is correctly removed as device_del
3889     # is async and empty return is not reliable
3890
3891     for (my $i = 0; $i <= 5; $i++) {
3892          my $devices_list = vm_devices_list($vmid);
3893          return 1 if !defined($devices_list->{$deviceid});
3894          sleep 1;
3895     }
3896
3897     die "error on hot-unplugging device '$deviceid'\n";
3898 }
3899
3900 sub qemu_findorcreatescsihw {
3901     my ($storecfg, $conf, $vmid, $device) = @_;
3902
3903     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3904
3905     my $scsihwid="$controller_prefix$controller";
3906     my $devices_list = vm_devices_list($vmid);
3907
3908     if(!defined($devices_list->{$scsihwid})) {
3909         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3910     }
3911
3912     return 1;
3913 }
3914
3915 sub qemu_deletescsihw {
3916     my ($conf, $vmid, $opt) = @_;
3917
3918     my $device = parse_drive($opt, $conf->{$opt});
3919
3920     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3921         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3922         return 1;
3923     }
3924
3925     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3926
3927     my $devices_list = vm_devices_list($vmid);
3928     foreach my $opt (keys %{$devices_list}) {
3929         if (PVE::QemuServer::is_valid_drivename($opt)) {
3930             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3931             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3932                 return 1;
3933             }
3934         }
3935     }
3936
3937     my $scsihwid="scsihw$controller";
3938
3939     vm_deviceunplug($vmid, $conf, $scsihwid);
3940
3941     return 1;
3942 }
3943
3944 sub qemu_add_pci_bridge {
3945     my ($storecfg, $conf, $vmid, $device) = @_;
3946
3947     my $bridges = {};
3948
3949     my $bridgeid;
3950
3951     print_pci_addr($device, $bridges);
3952
3953     while (my ($k, $v) = each %$bridges) {
3954         $bridgeid = $k;
3955     }
3956     return 1 if !defined($bridgeid) || $bridgeid < 1;
3957
3958     my $bridge = "pci.$bridgeid";
3959     my $devices_list = vm_devices_list($vmid);
3960
3961     if (!defined($devices_list->{$bridge})) {
3962         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3963     }
3964
3965     return 1;
3966 }
3967
3968 sub qemu_set_link_status {
3969     my ($vmid, $device, $up) = @_;
3970
3971     vm_mon_cmd($vmid, "set_link", name => $device,
3972                up => $up ? JSON::true : JSON::false);
3973 }
3974
3975 sub qemu_netdevadd {
3976     my ($vmid, $conf, $device, $deviceid) = @_;
3977
3978     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3979     my %options =  split(/[=,]/, $netdev);
3980
3981     vm_mon_cmd($vmid, "netdev_add",  %options);
3982     return 1;
3983 }
3984
3985 sub qemu_netdevdel {
3986     my ($vmid, $deviceid) = @_;
3987
3988     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3989 }
3990
3991 sub qemu_usb_hotplug {
3992     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3993
3994     return if !$device;
3995
3996     # remove the old one first
3997     vm_deviceunplug($vmid, $conf, $deviceid);
3998
3999     # check if xhci controller is necessary and available
4000     if ($device->{usb3}) {
4001
4002         my $devicelist = vm_devices_list($vmid);
4003
4004         if (!$devicelist->{xhci}) {
4005             my $pciaddr = print_pci_addr("xhci");
4006             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4007         }
4008     }
4009     my $d = parse_usb_device($device->{host});
4010     $d->{usb3} = $device->{usb3};
4011
4012     # add the new one
4013     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4014 }
4015
4016 sub qemu_cpu_hotplug {
4017     my ($vmid, $conf, $vcpus) = @_;
4018
4019     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4020
4021     my $sockets = 1;
4022     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4023     $sockets = $conf->{sockets} if  $conf->{sockets};
4024     my $cores = $conf->{cores} || 1;
4025     my $maxcpus = $sockets * $cores;
4026
4027     $vcpus = $maxcpus if !$vcpus;
4028
4029     die "you can't add more vcpus than maxcpus\n"
4030         if $vcpus > $maxcpus;
4031
4032     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4033
4034     if ($vcpus < $currentvcpus) {
4035
4036         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4037
4038             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4039                 qemu_devicedel($vmid, "cpu$i");
4040                 my $retry = 0;
4041                 my $currentrunningvcpus = undef;
4042                 while (1) {
4043                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4044                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4045                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4046                     $retry++;
4047                     sleep 1;
4048                 }
4049                 #update conf after each succesfull cpu unplug
4050                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4051                 PVE::QemuConfig->write_config($vmid, $conf);
4052             }
4053         } else {
4054             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4055         }
4056
4057         return;
4058     }
4059
4060     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4061     die "vcpus in running vm does not match its configuration\n"
4062         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4063
4064     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4065
4066         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4067             my $cpustr = print_cpu_device($conf, $i);
4068             qemu_deviceadd($vmid, $cpustr);
4069
4070             my $retry = 0;
4071             my $currentrunningvcpus = undef;
4072             while (1) {
4073                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4074                 last if scalar(@{$currentrunningvcpus}) == $i;
4075                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4076                 sleep 1;
4077                 $retry++;
4078             }
4079             #update conf after each succesfull cpu hotplug
4080             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4081             PVE::QemuConfig->write_config($vmid, $conf);
4082         }
4083     } else {
4084
4085         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4086             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4087         }
4088     }
4089 }
4090
4091 sub qemu_block_set_io_throttle {
4092     my ($vmid, $deviceid,
4093         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4094         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4095         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4096         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4097
4098     return if !check_running($vmid) ;
4099
4100     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4101         bps => int($bps),
4102         bps_rd => int($bps_rd),
4103         bps_wr => int($bps_wr),
4104         iops => int($iops),
4105         iops_rd => int($iops_rd),
4106         iops_wr => int($iops_wr),
4107         bps_max => int($bps_max),
4108         bps_rd_max => int($bps_rd_max),
4109         bps_wr_max => int($bps_wr_max),
4110         iops_max => int($iops_max),
4111         iops_rd_max => int($iops_rd_max),
4112         iops_wr_max => int($iops_wr_max),
4113         bps_max_length => int($bps_max_length),
4114         bps_rd_max_length => int($bps_rd_max_length),
4115         bps_wr_max_length => int($bps_wr_max_length),
4116         iops_max_length => int($iops_max_length),
4117         iops_rd_max_length => int($iops_rd_max_length),
4118         iops_wr_max_length => int($iops_wr_max_length),
4119     );
4120
4121 }
4122
4123 # old code, only used to shutdown old VM after update
4124 sub __read_avail {
4125     my ($fh, $timeout) = @_;
4126
4127     my $sel = new IO::Select;
4128     $sel->add($fh);
4129
4130     my $res = '';
4131     my $buf;
4132
4133     my @ready;
4134     while (scalar (@ready = $sel->can_read($timeout))) {
4135         my $count;
4136         if ($count = $fh->sysread($buf, 8192)) {
4137             if ($buf =~ /^(.*)\(qemu\) $/s) {
4138                 $res .= $1;
4139                 last;
4140             } else {
4141                 $res .= $buf;
4142             }
4143         } else {
4144             if (!defined($count)) {
4145                 die "$!\n";
4146             }
4147             last;
4148         }
4149     }
4150
4151     die "monitor read timeout\n" if !scalar(@ready);
4152
4153     return $res;
4154 }
4155
4156 # old code, only used to shutdown old VM after update
4157 sub vm_monitor_command {
4158     my ($vmid, $cmdstr, $nocheck) = @_;
4159
4160     my $res;
4161
4162     eval {
4163         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4164
4165         my $sname = "${var_run_tmpdir}/$vmid.mon";
4166
4167         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4168             die "unable to connect to VM $vmid socket - $!\n";
4169
4170         my $timeout = 3;
4171
4172         # hack: migrate sometime blocks the monitor (when migrate_downtime
4173         # is set)
4174         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4175             $timeout = 60*60; # 1 hour
4176         }
4177
4178         # read banner;
4179         my $data = __read_avail($sock, $timeout);
4180
4181         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4182             die "got unexpected qemu monitor banner\n";
4183         }
4184
4185         my $sel = new IO::Select;
4186         $sel->add($sock);
4187
4188         if (!scalar(my @ready = $sel->can_write($timeout))) {
4189             die "monitor write error - timeout";
4190         }
4191
4192         my $fullcmd = "$cmdstr\r";
4193
4194         # syslog('info', "VM $vmid monitor command: $cmdstr");
4195
4196         my $b;
4197         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4198             die "monitor write error - $!";
4199         }
4200
4201         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4202
4203         $timeout = 20;
4204
4205         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4206             $timeout = 60*60; # 1 hour
4207         } elsif ($cmdstr =~ m/^(eject|change)/) {
4208             $timeout = 60; # note: cdrom mount command is slow
4209         }
4210         if ($res = __read_avail($sock, $timeout)) {
4211
4212             my @lines = split("\r?\n", $res);
4213
4214             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4215
4216             $res = join("\n", @lines);
4217             $res .= "\n";
4218         }
4219     };
4220
4221     my $err = $@;
4222
4223     if ($err) {
4224         syslog("err", "VM $vmid monitor command failed - $err");
4225         die $err;
4226     }
4227
4228     return $res;
4229 }
4230
4231 sub qemu_block_resize {
4232     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4233
4234     my $running = check_running($vmid);
4235
4236     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4237
4238     return if !$running;
4239
4240     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4241
4242 }
4243
4244 sub qemu_volume_snapshot {
4245     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4246
4247     my $running = check_running($vmid);
4248
4249     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4250         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4251     } else {
4252         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4253     }
4254 }
4255
4256 sub qemu_volume_snapshot_delete {
4257     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4258
4259     my $running = check_running($vmid);
4260
4261     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4262         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4263     } else {
4264         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4265     }
4266 }
4267
4268 sub set_migration_caps {
4269     my ($vmid) = @_;
4270
4271     my $cap_ref = [];
4272
4273     my $enabled_cap = {
4274         "auto-converge" => 1,
4275         "xbzrle" => 1,
4276         "x-rdma-pin-all" => 0,
4277         "zero-blocks" => 0,
4278         "compress" => 0
4279     };
4280
4281     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4282
4283     for my $supported_capability (@$supported_capabilities) {
4284         push @$cap_ref, {
4285             capability => $supported_capability->{capability},
4286             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4287         };
4288     }
4289
4290     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4291 }
4292
4293 my $fast_plug_option = {
4294     'lock' => 1,
4295     'name' => 1,
4296     'onboot' => 1,
4297     'shares' => 1,
4298     'startup' => 1,
4299     'description' => 1,
4300     'protection' => 1,
4301     'vmstatestorage' => 1,
4302 };
4303
4304 # hotplug changes in [PENDING]
4305 # $selection hash can be used to only apply specified options, for
4306 # example: { cores => 1 } (only apply changed 'cores')
4307 # $errors ref is used to return error messages
4308 sub vmconfig_hotplug_pending {
4309     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4310
4311     my $defaults = load_defaults();
4312
4313     # commit values which do not have any impact on running VM first
4314     # Note: those option cannot raise errors, we we do not care about
4315     # $selection and always apply them.
4316
4317     my $add_error = sub {
4318         my ($opt, $msg) = @_;
4319         $errors->{$opt} = "hotplug problem - $msg";
4320     };
4321
4322     my $changes = 0;
4323     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4324         if ($fast_plug_option->{$opt}) {
4325             $conf->{$opt} = $conf->{pending}->{$opt};
4326             delete $conf->{pending}->{$opt};
4327             $changes = 1;
4328         }
4329     }
4330
4331     if ($changes) {
4332         PVE::QemuConfig->write_config($vmid, $conf);
4333         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4334     }
4335
4336     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4337
4338     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4339     while (my ($opt, $force) = each %$pending_delete_hash) {
4340         next if $selection && !$selection->{$opt};
4341         eval {
4342             if ($opt eq 'hotplug') {
4343                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4344             } elsif ($opt eq 'tablet') {
4345                 die "skip\n" if !$hotplug_features->{usb};
4346                 if ($defaults->{tablet}) {
4347                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4348                 } else {
4349                     vm_deviceunplug($vmid, $conf, $opt);
4350                 }
4351             } elsif ($opt =~ m/^usb\d+/) {
4352                 die "skip\n";
4353                 # since we cannot reliably hot unplug usb devices
4354                 # we are disabling it
4355                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4356                 vm_deviceunplug($vmid, $conf, $opt);
4357             } elsif ($opt eq 'vcpus') {
4358                 die "skip\n" if !$hotplug_features->{cpu};
4359                 qemu_cpu_hotplug($vmid, $conf, undef);
4360             } elsif ($opt eq 'balloon') {
4361                 # enable balloon device is not hotpluggable
4362                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4363             } elsif ($fast_plug_option->{$opt}) {
4364                 # do nothing
4365             } elsif ($opt =~ m/^net(\d+)$/) {
4366                 die "skip\n" if !$hotplug_features->{network};
4367                 vm_deviceunplug($vmid, $conf, $opt);
4368             } elsif (is_valid_drivename($opt)) {
4369                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4370                 vm_deviceunplug($vmid, $conf, $opt);
4371                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4372             } elsif ($opt =~ m/^memory$/) {
4373                 die "skip\n" if !$hotplug_features->{memory};
4374                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4375             } elsif ($opt eq 'cpuunits') {
4376                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4377             } elsif ($opt eq 'cpulimit') {
4378                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4379             } else {
4380                 die "skip\n";
4381             }
4382         };
4383         if (my $err = $@) {
4384             &$add_error($opt, $err) if $err ne "skip\n";
4385         } else {
4386             # save new config if hotplug was successful
4387             delete $conf->{$opt};
4388             vmconfig_undelete_pending_option($conf, $opt);
4389             PVE::QemuConfig->write_config($vmid, $conf);
4390             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4391         }
4392     }
4393
4394     my $apply_pending_cloudinit;
4395     $apply_pending_cloudinit = sub {
4396         my ($key, $value) = @_;
4397         $apply_pending_cloudinit = sub {}; # once is enough
4398
4399         my @cloudinit_opts = keys %$confdesc_cloudinit;
4400         foreach my $opt (keys %{$conf->{pending}}) {
4401             next if !grep { $_ eq $opt } @cloudinit_opts;
4402             $conf->{$opt} = delete $conf->{pending}->{$opt};
4403         }
4404
4405         my $new_conf = { %$conf };
4406         $new_conf->{$key} = $value;
4407         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4408     };
4409
4410     foreach my $opt (keys %{$conf->{pending}}) {
4411         next if $selection && !$selection->{$opt};
4412         my $value = $conf->{pending}->{$opt};
4413         eval {
4414             if ($opt eq 'hotplug') {
4415                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4416             } elsif ($opt eq 'tablet') {
4417                 die "skip\n" if !$hotplug_features->{usb};
4418                 if ($value == 1) {
4419                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4420                 } elsif ($value == 0) {
4421                     vm_deviceunplug($vmid, $conf, $opt);
4422                 }
4423             } elsif ($opt =~ m/^usb\d+$/) {
4424                 die "skip\n";
4425                 # since we cannot reliably hot unplug usb devices
4426                 # we are disabling it
4427                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4428                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4429                 die "skip\n" if !$d;
4430                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4431             } elsif ($opt eq 'vcpus') {
4432                 die "skip\n" if !$hotplug_features->{cpu};
4433                 qemu_cpu_hotplug($vmid, $conf, $value);
4434             } elsif ($opt eq 'balloon') {
4435                 # enable/disable balloning device is not hotpluggable
4436                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4437                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4438                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4439
4440                 # allow manual ballooning if shares is set to zero
4441                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4442                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4443                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4444                 }
4445             } elsif ($opt =~ m/^net(\d+)$/) {
4446                 # some changes can be done without hotplug
4447                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4448                                     $vmid, $opt, $value);
4449             } elsif (is_valid_drivename($opt)) {
4450                 # some changes can be done without hotplug
4451                 my $drive = parse_drive($opt, $value);
4452                 if (drive_is_cloudinit($drive)) {
4453                     &$apply_pending_cloudinit($opt, $value);
4454                 }
4455                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4456                                      $vmid, $opt, $value, 1);
4457             } elsif ($opt =~ m/^memory$/) { #dimms
4458                 die "skip\n" if !$hotplug_features->{memory};
4459                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4460             } elsif ($opt eq 'cpuunits') {
4461                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4462             } elsif ($opt eq 'cpulimit') {
4463                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4464                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4465             } else {
4466                 die "skip\n";  # skip non-hot-pluggable options
4467             }
4468         };
4469         if (my $err = $@) {
4470             &$add_error($opt, $err) if $err ne "skip\n";
4471         } else {
4472             # save new config if hotplug was successful
4473             $conf->{$opt} = $value;
4474             delete $conf->{pending}->{$opt};
4475             PVE::QemuConfig->write_config($vmid, $conf);
4476             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4477         }
4478     }
4479 }
4480
4481 sub try_deallocate_drive {
4482     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4483
4484     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4485         my $volid = $drive->{file};
4486         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4487             my $sid = PVE::Storage::parse_volume_id($volid);
4488             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4489
4490             # check if the disk is really unused
4491             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4492                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4493             PVE::Storage::vdisk_free($storecfg, $volid);
4494             return 1;
4495         } else {
4496             # If vm is not owner of this disk remove from config
4497             return 1;
4498         }
4499     }
4500
4501     return undef;
4502 }
4503
4504 sub vmconfig_delete_or_detach_drive {
4505     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4506
4507     my $drive = parse_drive($opt, $conf->{$opt});
4508
4509     my $rpcenv = PVE::RPCEnvironment::get();
4510     my $authuser = $rpcenv->get_user();
4511
4512     if ($force) {
4513         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4514         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4515     } else {
4516         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4517     }
4518 }
4519
4520 sub vmconfig_apply_pending {
4521     my ($vmid, $conf, $storecfg) = @_;
4522
4523     # cold plug
4524
4525     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4526     while (my ($opt, $force) = each %$pending_delete_hash) {
4527         die "internal error" if $opt =~ m/^unused/;
4528         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4529         if (!defined($conf->{$opt})) {
4530             vmconfig_undelete_pending_option($conf, $opt);
4531             PVE::QemuConfig->write_config($vmid, $conf);
4532         } elsif (is_valid_drivename($opt)) {
4533             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4534             vmconfig_undelete_pending_option($conf, $opt);
4535             delete $conf->{$opt};
4536             PVE::QemuConfig->write_config($vmid, $conf);
4537         } else {
4538             vmconfig_undelete_pending_option($conf, $opt);
4539             delete $conf->{$opt};
4540             PVE::QemuConfig->write_config($vmid, $conf);
4541         }
4542     }
4543
4544     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4545
4546     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4547         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4548
4549         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4550             # skip if nothing changed
4551         } elsif (is_valid_drivename($opt)) {
4552             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4553                 if defined($conf->{$opt});
4554             $conf->{$opt} = $conf->{pending}->{$opt};
4555         } else {
4556             $conf->{$opt} = $conf->{pending}->{$opt};
4557         }
4558
4559         delete $conf->{pending}->{$opt};
4560         PVE::QemuConfig->write_config($vmid, $conf);
4561     }
4562 }
4563
4564 my $safe_num_ne = sub {
4565     my ($a, $b) = @_;
4566
4567     return 0 if !defined($a) && !defined($b);
4568     return 1 if !defined($a);
4569     return 1 if !defined($b);
4570
4571     return $a != $b;
4572 };
4573
4574 my $safe_string_ne = sub {
4575     my ($a, $b) = @_;
4576
4577     return 0 if !defined($a) && !defined($b);
4578     return 1 if !defined($a);
4579     return 1 if !defined($b);
4580
4581     return $a ne $b;
4582 };
4583
4584 sub vmconfig_update_net {
4585     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4586
4587     my $newnet = parse_net($value);
4588
4589     if ($conf->{$opt}) {
4590         my $oldnet = parse_net($conf->{$opt});
4591
4592         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4593             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4594             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4595             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4596
4597             # for non online change, we try to hot-unplug
4598             die "skip\n" if !$hotplug;
4599             vm_deviceunplug($vmid, $conf, $opt);
4600         } else {
4601
4602             die "internal error" if $opt !~ m/net(\d+)/;
4603             my $iface = "tap${vmid}i$1";
4604
4605             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4606                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4607                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4608                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4609                 PVE::Network::tap_unplug($iface);
4610                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4611             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4612                 # Rate can be applied on its own but any change above needs to
4613                 # include the rate in tap_plug since OVS resets everything.
4614                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4615             }
4616
4617             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4618                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4619             }
4620
4621             return 1;
4622         }
4623     }
4624
4625     if ($hotplug) {
4626         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4627     } else {
4628         die "skip\n";
4629     }
4630 }
4631
4632 sub vmconfig_update_disk {
4633     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4634
4635     # fixme: do we need force?
4636
4637     my $drive = parse_drive($opt, $value);
4638
4639     if ($conf->{$opt}) {
4640
4641         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4642
4643             my $media = $drive->{media} || 'disk';
4644             my $oldmedia = $old_drive->{media} || 'disk';
4645             die "unable to change media type\n" if $media ne $oldmedia;
4646
4647             if (!drive_is_cdrom($old_drive)) {
4648
4649                 if ($drive->{file} ne $old_drive->{file}) {
4650
4651                     die "skip\n" if !$hotplug;
4652
4653                     # unplug and register as unused
4654                     vm_deviceunplug($vmid, $conf, $opt);
4655                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4656
4657                 } else {
4658                     # update existing disk
4659
4660                     # skip non hotpluggable value
4661                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4662                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4663                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4664                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4665                         die "skip\n";
4666                     }
4667
4668                     # apply throttle
4669                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4670                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4671                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4672                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4673                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4674                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4675                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4676                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4677                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4678                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4679                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4680                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4681                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4682                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4683                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4684                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4685                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4686                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4687
4688                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4689                                                    ($drive->{mbps} || 0)*1024*1024,
4690                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4691                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4692                                                    $drive->{iops} || 0,
4693                                                    $drive->{iops_rd} || 0,
4694                                                    $drive->{iops_wr} || 0,
4695                                                    ($drive->{mbps_max} || 0)*1024*1024,
4696                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4697                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4698                                                    $drive->{iops_max} || 0,
4699                                                    $drive->{iops_rd_max} || 0,
4700                                                    $drive->{iops_wr_max} || 0,
4701                                                    $drive->{bps_max_length} || 1,
4702                                                    $drive->{bps_rd_max_length} || 1,
4703                                                    $drive->{bps_wr_max_length} || 1,
4704                                                    $drive->{iops_max_length} || 1,
4705                                                    $drive->{iops_rd_max_length} || 1,
4706                                                    $drive->{iops_wr_max_length} || 1);
4707
4708                     }
4709
4710                     return 1;
4711                 }
4712
4713             } else { # cdrom
4714
4715                 if ($drive->{file} eq 'none') {
4716                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4717                     if (drive_is_cloudinit($old_drive)) {
4718                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4719                     }
4720                 } else {
4721                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4722                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4723                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4724                 }
4725
4726                 return 1;
4727             }
4728         }
4729     }
4730
4731     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4732     # hotplug new disks
4733     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4734     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4735 }
4736
4737 sub vm_start {
4738     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4739         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4740
4741     PVE::QemuConfig->lock_config($vmid, sub {
4742         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4743
4744         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4745
4746         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4747
4748         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4749
4750         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4751             vmconfig_apply_pending($vmid, $conf, $storecfg);
4752             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4753         }
4754
4755         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4756
4757         my $defaults = load_defaults();
4758
4759         # set environment variable useful inside network script
4760         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4761
4762         my $local_volumes = {};
4763
4764         if ($targetstorage) {
4765             foreach_drive($conf, sub {
4766                 my ($ds, $drive) = @_;
4767
4768                 return if drive_is_cdrom($drive);
4769
4770                 my $volid = $drive->{file};
4771
4772                 return if !$volid;
4773
4774                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4775
4776                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4777                 return if $scfg->{shared};
4778                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4779             });
4780
4781             my $format = undef;
4782
4783             foreach my $opt (sort keys %$local_volumes) {
4784
4785                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4786                 my $drive = parse_drive($opt, $conf->{$opt});
4787
4788                 #if remote storage is specified, use default format
4789                 if ($targetstorage && $targetstorage ne "1") {
4790                     $storeid = $targetstorage;
4791                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4792                     $format = $defFormat;
4793                 } else {
4794                     #else we use same format than original
4795                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4796                     $format = qemu_img_format($scfg, $volid);
4797                 }
4798
4799                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4800                 my $newdrive = $drive;
4801                 $newdrive->{format} = $format;
4802                 $newdrive->{file} = $newvolid;
4803                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4804                 $local_volumes->{$opt} = $drivestr;
4805                 #pass drive to conf for command line
4806                 $conf->{$opt} = $drivestr;
4807             }
4808         }
4809
4810         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4811
4812         my $migrate_port = 0;
4813         my $migrate_uri;
4814         if ($statefile) {
4815             if ($statefile eq 'tcp') {
4816                 my $localip = "localhost";
4817                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4818                 my $nodename = PVE::INotify::nodename();
4819
4820                 if (!defined($migration_type)) {
4821                     if (defined($datacenterconf->{migration}->{type})) {
4822                         $migration_type = $datacenterconf->{migration}->{type};
4823                     } else {
4824                         $migration_type = 'secure';
4825                     }
4826                 }
4827
4828                 if ($migration_type eq 'insecure') {
4829                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4830                     if ($migrate_network_addr) {
4831                         $localip = $migrate_network_addr;
4832                     } else {
4833                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4834                     }
4835
4836                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4837                 }
4838
4839                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4840                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4841                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4842                 push @$cmd, '-incoming', $migrate_uri;
4843                 push @$cmd, '-S';
4844
4845             } elsif ($statefile eq 'unix') {
4846                 # should be default for secure migrations as a ssh TCP forward
4847                 # tunnel is not deterministic reliable ready and fails regurarly
4848                 # to set up in time, so use UNIX socket forwards
4849                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4850                 unlink $socket_addr;
4851
4852                 $migrate_uri = "unix:$socket_addr";
4853
4854                 push @$cmd, '-incoming', $migrate_uri;
4855                 push @$cmd, '-S';
4856
4857             } else {
4858                 push @$cmd, '-loadstate', $statefile;
4859             }
4860         } elsif ($paused) {
4861             push @$cmd, '-S';
4862         }
4863
4864         # host pci devices
4865         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4866           my $d = parse_hostpci($conf->{"hostpci$i"});
4867           next if !$d;
4868           my $pcidevices = $d->{pciid};
4869           foreach my $pcidevice (@$pcidevices) {
4870                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4871
4872                 my $info = pci_device_info("0000:$pciid");
4873                 die "IOMMU not present\n" if !check_iommu_support();
4874                 die "no pci device info for device '$pciid'\n" if !$info;
4875                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4876                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4877           }
4878         }
4879
4880         PVE::Storage::activate_volumes($storecfg, $vollist);
4881
4882         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4883             my $cmd = [];
4884             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4885             eval  { run_command($cmd); };
4886         }
4887
4888         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4889                                                   : $defaults->{cpuunits};
4890
4891         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4892         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4893
4894         my %properties = (
4895             Slice => 'qemu.slice',
4896             KillMode => 'none',
4897             CPUShares => $cpuunits
4898         );
4899
4900         if (my $cpulimit = $conf->{cpulimit}) {
4901             $properties{CPUQuota} = int($cpulimit * 100);
4902         }
4903         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4904
4905         if ($conf->{hugepages}) {
4906
4907             my $code = sub {
4908                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4909                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4910
4911                 PVE::QemuServer::Memory::hugepages_mount();
4912                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4913
4914                 eval  {
4915                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4916                     run_command($cmd, %run_params);
4917                 };
4918
4919                 if (my $err = $@) {
4920                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4921                     die $err;
4922                 }
4923
4924                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4925             };
4926             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4927
4928         } else {
4929             eval  {
4930                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4931                 run_command($cmd, %run_params);
4932             };
4933         }
4934
4935         if (my $err = $@) {
4936             # deactivate volumes if start fails
4937             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4938             die "start failed: $err";
4939         }
4940
4941         print "migration listens on $migrate_uri\n" if $migrate_uri;
4942
4943         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4944             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4945             warn $@ if $@;
4946         }
4947
4948         #start nbd server for storage migration
4949         if ($targetstorage) {
4950             my $nodename = PVE::INotify::nodename();
4951             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4952             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4953             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4954             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4955
4956             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4957
4958             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4959
4960             foreach my $opt (sort keys %$local_volumes) {
4961                 my $volid = $local_volumes->{$opt};
4962                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4963                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4964                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4965             }
4966         }
4967
4968         if ($migratedfrom) {
4969             eval {
4970                 set_migration_caps($vmid);
4971             };
4972             warn $@ if $@;
4973
4974             if ($spice_port) {
4975                 print "spice listens on port $spice_port\n";
4976                 if ($spice_ticket) {
4977                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4978                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4979                 }
4980             }
4981
4982         } else {
4983             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4984                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4985                     if $conf->{balloon};
4986             }
4987
4988             foreach my $opt (keys %$conf) {
4989                 next if $opt !~  m/^net\d+$/;
4990                 my $nicconf = parse_net($conf->{$opt});
4991                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4992             }
4993         }
4994
4995         vm_mon_cmd_nocheck($vmid, 'qom-set',
4996                     path => "machine/peripheral/balloon0",
4997                     property => "guest-stats-polling-interval",
4998                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4999
5000     });
5001 }
5002
5003 sub vm_mon_cmd {
5004     my ($vmid, $execute, %params) = @_;
5005
5006     my $cmd = { execute => $execute, arguments => \%params };
5007     vm_qmp_command($vmid, $cmd);
5008 }
5009
5010 sub vm_mon_cmd_nocheck {
5011     my ($vmid, $execute, %params) = @_;
5012
5013     my $cmd = { execute => $execute, arguments => \%params };
5014     vm_qmp_command($vmid, $cmd, 1);
5015 }
5016
5017 sub vm_qmp_command {
5018     my ($vmid, $cmd, $nocheck) = @_;
5019
5020     my $res;
5021
5022     my $timeout;
5023     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5024         $timeout = $cmd->{arguments}->{timeout};
5025         delete $cmd->{arguments}->{timeout};
5026     }
5027
5028     eval {
5029         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5030         my $sname = qmp_socket($vmid);
5031         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5032             my $qmpclient = PVE::QMPClient->new();
5033
5034             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5035         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
5036             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
5037                 if scalar(%{$cmd->{arguments}});
5038             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
5039         } else {
5040             die "unable to open monitor socket\n";
5041         }
5042     };
5043     if (my $err = $@) {
5044         syslog("err", "VM $vmid qmp command failed - $err");
5045         die $err;
5046     }
5047
5048     return $res;
5049 }
5050
5051 sub vm_human_monitor_command {
5052     my ($vmid, $cmdline) = @_;
5053
5054     my $res;
5055
5056     my $cmd = {
5057         execute => 'human-monitor-command',
5058         arguments => { 'command-line' => $cmdline},
5059     };
5060
5061     return vm_qmp_command($vmid, $cmd);
5062 }
5063
5064 sub vm_commandline {
5065     my ($storecfg, $vmid) = @_;
5066
5067     my $conf = PVE::QemuConfig->load_config($vmid);
5068
5069     my $defaults = load_defaults();
5070
5071     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5072
5073     return PVE::Tools::cmd2string($cmd);
5074 }
5075
5076 sub vm_reset {
5077     my ($vmid, $skiplock) = @_;
5078
5079     PVE::QemuConfig->lock_config($vmid, sub {
5080
5081         my $conf = PVE::QemuConfig->load_config($vmid);
5082
5083         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5084
5085         vm_mon_cmd($vmid, "system_reset");
5086     });
5087 }
5088
5089 sub get_vm_volumes {
5090     my ($conf) = @_;
5091
5092     my $vollist = [];
5093     foreach_volid($conf, sub {
5094         my ($volid, $attr) = @_;
5095
5096         return if $volid =~ m|^/|;
5097
5098         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5099         return if !$sid;
5100
5101         push @$vollist, $volid;
5102     });
5103
5104     return $vollist;
5105 }
5106
5107 sub vm_stop_cleanup {
5108     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5109
5110     eval {
5111
5112         if (!$keepActive) {
5113             my $vollist = get_vm_volumes($conf);
5114             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5115         }
5116
5117         foreach my $ext (qw(mon qmp pid vnc qga)) {
5118             unlink "/var/run/qemu-server/${vmid}.$ext";
5119         }
5120
5121         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5122     };
5123     warn $@ if $@; # avoid errors - just warn
5124 }
5125
5126 # Note: use $nockeck to skip tests if VM configuration file exists.
5127 # We need that when migration VMs to other nodes (files already moved)
5128 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5129 sub vm_stop {
5130     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5131
5132     $force = 1 if !defined($force) && !$shutdown;
5133
5134     if ($migratedfrom){
5135         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5136         kill 15, $pid if $pid;
5137         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5138         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5139         return;
5140     }
5141
5142     PVE::QemuConfig->lock_config($vmid, sub {
5143
5144         my $pid = check_running($vmid, $nocheck);
5145         return if !$pid;
5146
5147         my $conf;
5148         if (!$nocheck) {
5149             $conf = PVE::QemuConfig->load_config($vmid);
5150             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5151             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5152                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5153                 $timeout = $opts->{down} if $opts->{down};
5154             }
5155         }
5156
5157         $timeout = 60 if !defined($timeout);
5158
5159         eval {
5160             if ($shutdown) {
5161                 if (defined($conf) && $conf->{agent}) {
5162                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5163                 } else {
5164                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5165                 }
5166             } else {
5167                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5168             }
5169         };
5170         my $err = $@;
5171
5172         if (!$err) {
5173             my $count = 0;
5174             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5175                 $count++;
5176                 sleep 1;
5177             }
5178
5179             if ($count >= $timeout) {
5180                 if ($force) {
5181                     warn "VM still running - terminating now with SIGTERM\n";
5182                     kill 15, $pid;
5183                 } else {
5184                     die "VM quit/powerdown failed - got timeout\n";
5185                 }
5186             } else {
5187                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5188                 return;
5189             }
5190         } else {
5191             if ($force) {
5192                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5193                 kill 15, $pid;
5194             } else {
5195                 die "VM quit/powerdown failed\n";
5196             }
5197         }
5198
5199         # wait again
5200         $timeout = 10;
5201
5202         my $count = 0;
5203         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5204             $count++;
5205             sleep 1;
5206         }
5207
5208         if ($count >= $timeout) {
5209             warn "VM still running - terminating now with SIGKILL\n";
5210             kill 9, $pid;
5211             sleep 1;
5212         }
5213
5214         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5215    });
5216 }
5217
5218 sub vm_suspend {
5219     my ($vmid, $skiplock) = @_;
5220
5221     PVE::QemuConfig->lock_config($vmid, sub {
5222
5223         my $conf = PVE::QemuConfig->load_config($vmid);
5224
5225         PVE::QemuConfig->check_lock($conf)
5226             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5227
5228         vm_mon_cmd($vmid, "stop");
5229     });
5230 }
5231
5232 sub vm_resume {
5233     my ($vmid, $skiplock, $nocheck) = @_;
5234
5235     PVE::QemuConfig->lock_config($vmid, sub {
5236
5237         if (!$nocheck) {
5238
5239             my $conf = PVE::QemuConfig->load_config($vmid);
5240
5241             PVE::QemuConfig->check_lock($conf)
5242                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5243
5244             vm_mon_cmd($vmid, "cont");
5245
5246         } else {
5247             vm_mon_cmd_nocheck($vmid, "cont");
5248         }
5249     });
5250 }
5251
5252 sub vm_sendkey {
5253     my ($vmid, $skiplock, $key) = @_;
5254
5255     PVE::QemuConfig->lock_config($vmid, sub {
5256
5257         my $conf = PVE::QemuConfig->load_config($vmid);
5258
5259         # there is no qmp command, so we use the human monitor command
5260         vm_human_monitor_command($vmid, "sendkey $key");
5261     });
5262 }
5263
5264 sub vm_destroy {
5265     my ($storecfg, $vmid, $skiplock) = @_;
5266
5267     PVE::QemuConfig->lock_config($vmid, sub {
5268
5269         my $conf = PVE::QemuConfig->load_config($vmid);
5270
5271         if (!check_running($vmid)) {
5272             destroy_vm($storecfg, $vmid, undef, $skiplock);
5273         } else {
5274             die "VM $vmid is running - destroy failed\n";
5275         }
5276     });
5277 }
5278
5279 # pci helpers
5280
5281 sub file_write {
5282     my ($filename, $buf) = @_;
5283
5284     my $fh = IO::File->new($filename, "w");
5285     return undef if !$fh;
5286
5287     my $res = print $fh $buf;
5288
5289     $fh->close();
5290
5291     return $res;
5292 }
5293
5294 sub pci_device_info {
5295     my ($name) = @_;
5296
5297     my $res;
5298
5299     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5300     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5301
5302     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5303     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5304
5305     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5306     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5307
5308     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5309     return undef if !defined($product) || $product !~ s/^0x//;
5310
5311     $res = {
5312         name => $name,
5313         vendor => $vendor,
5314         product => $product,
5315         domain => $domain,
5316         bus => $bus,
5317         slot => $slot,
5318         func => $func,
5319         irq => $irq,
5320         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5321     };
5322
5323     return $res;
5324 }
5325
5326 sub pci_dev_reset {
5327     my ($dev) = @_;
5328
5329     my $name = $dev->{name};
5330
5331     my $fn = "$pcisysfs/devices/$name/reset";
5332
5333     return file_write($fn, "1");
5334 }
5335
5336 sub pci_dev_bind_to_vfio {
5337     my ($dev) = @_;
5338
5339     my $name = $dev->{name};
5340
5341     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5342
5343     if (!-d $vfio_basedir) {
5344         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5345     }
5346     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5347
5348     my $testdir = "$vfio_basedir/$name";
5349     return 1 if -d $testdir;
5350
5351     my $data = "$dev->{vendor} $dev->{product}";
5352     return undef if !file_write("$vfio_basedir/new_id", $data);
5353
5354     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5355     if (!file_write($fn, $name)) {
5356         return undef if -f $fn;
5357     }
5358
5359     $fn = "$vfio_basedir/bind";
5360     if (! -d $testdir) {
5361         return undef if !file_write($fn, $name);
5362     }
5363
5364     return -d $testdir;
5365 }
5366
5367 sub pci_dev_group_bind_to_vfio {
5368     my ($pciid) = @_;
5369
5370     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5371
5372     if (!-d $vfio_basedir) {
5373         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5374     }
5375     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5376
5377     # get IOMMU group devices
5378     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5379       my @devs = grep /^0000:/, readdir($D);
5380     closedir($D);
5381
5382     foreach my $pciid (@devs) {
5383         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5384
5385         # pci bridges, switches or root ports are not supported
5386         # they have a pci_bus subdirectory so skip them
5387         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5388
5389         my $info = pci_device_info($1);
5390         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5391     }
5392
5393     return 1;
5394 }
5395
5396 # vzdump restore implementaion
5397
5398 sub tar_archive_read_firstfile {
5399     my $archive = shift;
5400
5401     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5402
5403     # try to detect archive type first
5404     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5405         die "unable to open file '$archive'\n";
5406     my $firstfile = <$fh>;
5407     kill 15, $pid;
5408     close $fh;
5409
5410     die "ERROR: archive contaions no data\n" if !$firstfile;
5411     chomp $firstfile;
5412
5413     return $firstfile;
5414 }
5415
5416 sub tar_restore_cleanup {
5417     my ($storecfg, $statfile) = @_;
5418
5419     print STDERR "starting cleanup\n";
5420
5421     if (my $fd = IO::File->new($statfile, "r")) {
5422         while (defined(my $line = <$fd>)) {
5423             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5424                 my $volid = $2;
5425                 eval {
5426                     if ($volid =~ m|^/|) {
5427                         unlink $volid || die 'unlink failed\n';
5428                     } else {
5429                         PVE::Storage::vdisk_free($storecfg, $volid);
5430                     }
5431                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5432                 };
5433                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5434             } else {
5435                 print STDERR "unable to parse line in statfile - $line";
5436             }
5437         }
5438         $fd->close();
5439     }
5440 }
5441
5442 sub restore_archive {
5443     my ($archive, $vmid, $user, $opts) = @_;
5444
5445     my $format = $opts->{format};
5446     my $comp;
5447
5448     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5449         $format = 'tar' if !$format;
5450         $comp = 'gzip';
5451     } elsif ($archive =~ m/\.tar$/) {
5452         $format = 'tar' if !$format;
5453     } elsif ($archive =~ m/.tar.lzo$/) {
5454         $format = 'tar' if !$format;
5455         $comp = 'lzop';
5456     } elsif ($archive =~ m/\.vma$/) {
5457         $format = 'vma' if !$format;
5458     } elsif ($archive =~ m/\.vma\.gz$/) {
5459         $format = 'vma' if !$format;
5460         $comp = 'gzip';
5461     } elsif ($archive =~ m/\.vma\.lzo$/) {
5462         $format = 'vma' if !$format;
5463         $comp = 'lzop';
5464     } else {
5465         $format = 'vma' if !$format; # default
5466     }
5467
5468     # try to detect archive format
5469     if ($format eq 'tar') {
5470         return restore_tar_archive($archive, $vmid, $user, $opts);
5471     } else {
5472         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5473     }
5474 }
5475
5476 sub restore_update_config_line {
5477     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5478
5479     return if $line =~ m/^\#qmdump\#/;
5480     return if $line =~ m/^\#vzdump\#/;
5481     return if $line =~ m/^lock:/;
5482     return if $line =~ m/^unused\d+:/;
5483     return if $line =~ m/^parent:/;
5484     return if $line =~ m/^template:/; # restored VM is never a template
5485
5486     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5487     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5488         # try to convert old 1.X settings
5489         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5490         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5491             my ($model, $macaddr) = split(/\=/, $devconfig);
5492             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5493             my $net = {
5494                 model => $model,
5495                 bridge => "vmbr$ind",
5496                 macaddr => $macaddr,
5497             };
5498             my $netstr = print_net($net);
5499
5500             print $outfd "net$cookie->{netcount}: $netstr\n";
5501             $cookie->{netcount}++;
5502         }
5503     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5504         my ($id, $netstr) = ($1, $2);
5505         my $net = parse_net($netstr);
5506         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5507         $netstr = print_net($net);
5508         print $outfd "$id: $netstr\n";
5509     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5510         my $virtdev = $1;
5511         my $value = $3;
5512         my $di = parse_drive($virtdev, $value);
5513         if (defined($di->{backup}) && !$di->{backup}) {
5514             print $outfd "#$line";
5515         } elsif ($map->{$virtdev}) {
5516             delete $di->{format}; # format can change on restore
5517             $di->{file} = $map->{$virtdev};
5518             $value = print_drive($vmid, $di);
5519             print $outfd "$virtdev: $value\n";
5520         } else {
5521             print $outfd $line;
5522         }
5523     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5524         my ($uuid, $uuid_str);
5525         UUID::generate($uuid);
5526         UUID::unparse($uuid, $uuid_str);
5527         my $smbios1 = parse_smbios1($2);
5528         $smbios1->{uuid} = $uuid_str;
5529         print $outfd $1.print_smbios1($smbios1)."\n";
5530     } else {
5531         print $outfd $line;
5532     }
5533 }
5534
5535 sub scan_volids {
5536     my ($cfg, $vmid) = @_;
5537
5538     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5539
5540     my $volid_hash = {};
5541     foreach my $storeid (keys %$info) {
5542         foreach my $item (@{$info->{$storeid}}) {
5543             next if !($item->{volid} && $item->{size});
5544             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5545             $volid_hash->{$item->{volid}} = $item;
5546         }
5547     }
5548
5549     return $volid_hash;
5550 }
5551
5552 sub is_volume_in_use {
5553     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5554
5555     my $path = PVE::Storage::path($storecfg, $volid);
5556
5557     my $scan_config = sub {
5558         my ($cref, $snapname) = @_;
5559
5560         foreach my $key (keys %$cref) {
5561             my $value = $cref->{$key};
5562             if (is_valid_drivename($key)) {
5563                 next if $skip_drive && $key eq $skip_drive;
5564                 my $drive = parse_drive($key, $value);
5565                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5566                 return 1 if $volid eq $drive->{file};
5567                 if ($drive->{file} =~ m!^/!) {
5568                     return 1 if $drive->{file} eq $path;
5569                 } else {
5570                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5571                     next if !$storeid;
5572                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5573                     next if !$scfg;
5574                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5575                 }
5576             }
5577         }
5578
5579         return 0;
5580     };
5581
5582     return 1 if &$scan_config($conf);
5583
5584     undef $skip_drive;
5585
5586     foreach my $snapname (keys %{$conf->{snapshots}}) {
5587         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5588     }
5589
5590     return 0;
5591 }
5592
5593 sub update_disksize {
5594     my ($vmid, $conf, $volid_hash) = @_;
5595
5596     my $changes;
5597
5598     # used and unused disks
5599     my $referenced = {};
5600
5601     # Note: it is allowed to define multiple storages with same path (alias), so
5602     # we need to check both 'volid' and real 'path' (two different volid can point
5603     # to the same path).
5604
5605     my $referencedpath = {};
5606
5607     # update size info
5608     foreach my $opt (keys %$conf) {
5609         if (is_valid_drivename($opt)) {
5610             my $drive = parse_drive($opt, $conf->{$opt});
5611             my $volid = $drive->{file};
5612             next if !$volid;
5613
5614             $referenced->{$volid} = 1;
5615             if ($volid_hash->{$volid} &&
5616                 (my $path = $volid_hash->{$volid}->{path})) {
5617                 $referencedpath->{$path} = 1;
5618             }
5619
5620             next if drive_is_cdrom($drive);
5621             next if !$volid_hash->{$volid};
5622
5623             $drive->{size} = $volid_hash->{$volid}->{size};
5624             my $new = print_drive($vmid, $drive);
5625             if ($new ne $conf->{$opt}) {
5626                 $changes = 1;
5627                 $conf->{$opt} = $new;
5628             }
5629         }
5630     }
5631
5632     # remove 'unusedX' entry if volume is used
5633     foreach my $opt (keys %$conf) {
5634         next if $opt !~ m/^unused\d+$/;
5635         my $volid = $conf->{$opt};
5636         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5637         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5638             $changes = 1;
5639             delete $conf->{$opt};
5640         }
5641
5642         $referenced->{$volid} = 1;
5643         $referencedpath->{$path} = 1 if $path;
5644     }
5645
5646     foreach my $volid (sort keys %$volid_hash) {
5647         next if $volid =~ m/vm-$vmid-state-/;
5648         next if $referenced->{$volid};
5649         my $path = $volid_hash->{$volid}->{path};
5650         next if !$path; # just to be sure
5651         next if $referencedpath->{$path};
5652         $changes = 1;
5653         PVE::QemuConfig->add_unused_volume($conf, $volid);
5654         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5655     }
5656
5657     return $changes;
5658 }
5659
5660 sub rescan {
5661     my ($vmid, $nolock) = @_;
5662
5663     my $cfg = PVE::Storage::config();
5664
5665     my $volid_hash = scan_volids($cfg, $vmid);
5666
5667     my $updatefn =  sub {
5668         my ($vmid) = @_;
5669
5670         my $conf = PVE::QemuConfig->load_config($vmid);
5671
5672         PVE::QemuConfig->check_lock($conf);
5673
5674         my $vm_volids = {};
5675         foreach my $volid (keys %$volid_hash) {
5676             my $info = $volid_hash->{$volid};
5677             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5678         }
5679
5680         my $changes = update_disksize($vmid, $conf, $vm_volids);
5681
5682         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5683     };
5684
5685     if (defined($vmid)) {
5686         if ($nolock) {
5687             &$updatefn($vmid);
5688         } else {
5689             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5690         }
5691     } else {
5692         my $vmlist = config_list();
5693         foreach my $vmid (keys %$vmlist) {
5694             if ($nolock) {
5695                 &$updatefn($vmid);
5696             } else {
5697                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5698             }
5699         }
5700     }
5701 }
5702
5703 sub restore_vma_archive {
5704     my ($archive, $vmid, $user, $opts, $comp) = @_;
5705
5706     my $input = $archive eq '-' ? "<&STDIN" : undef;
5707     my $readfrom = $archive;
5708
5709     my $uncomp = '';
5710     if ($comp) {
5711         $readfrom = '-';
5712         my $qarchive = PVE::Tools::shellquote($archive);
5713         if ($comp eq 'gzip') {
5714             $uncomp = "zcat $qarchive|";
5715         } elsif ($comp eq 'lzop') {
5716             $uncomp = "lzop -d -c $qarchive|";
5717         } else {
5718             die "unknown compression method '$comp'\n";
5719         }
5720
5721     }
5722
5723     my $tmpdir = "/var/tmp/vzdumptmp$$";
5724     rmtree $tmpdir;
5725
5726     # disable interrupts (always do cleanups)
5727     local $SIG{INT} =
5728         local $SIG{TERM} =
5729         local $SIG{QUIT} =
5730         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5731
5732     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5733     POSIX::mkfifo($mapfifo, 0600);
5734     my $fifofh;
5735
5736     my $openfifo = sub {
5737         open($fifofh, '>', $mapfifo) || die $!;
5738     };
5739
5740     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5741
5742     my $oldtimeout;
5743     my $timeout = 5;
5744
5745     my $devinfo = {};
5746
5747     my $rpcenv = PVE::RPCEnvironment::get();
5748
5749     my $conffile = PVE::QemuConfig->config_file($vmid);
5750     my $tmpfn = "$conffile.$$.tmp";
5751
5752     # Note: $oldconf is undef if VM does not exists
5753     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5754     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5755
5756     my $print_devmap = sub {
5757         my $virtdev_hash = {};
5758
5759         my $cfgfn = "$tmpdir/qemu-server.conf";
5760
5761         # we can read the config - that is already extracted
5762         my $fh = IO::File->new($cfgfn, "r") ||
5763             "unable to read qemu-server.conf - $!\n";
5764
5765         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5766         if (-f $fwcfgfn) {
5767             my $pve_firewall_dir = '/etc/pve/firewall';
5768             mkdir $pve_firewall_dir; # make sure the dir exists
5769             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5770         }
5771
5772         while (defined(my $line = <$fh>)) {
5773             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5774                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5775                 die "archive does not contain data for drive '$virtdev'\n"
5776                     if !$devinfo->{$devname};
5777                 if (defined($opts->{storage})) {
5778                     $storeid = $opts->{storage} || 'local';
5779                 } elsif (!$storeid) {
5780                     $storeid = 'local';
5781                 }
5782                 $format = 'raw' if !$format;
5783                 $devinfo->{$devname}->{devname} = $devname;
5784                 $devinfo->{$devname}->{virtdev} = $virtdev;
5785                 $devinfo->{$devname}->{format} = $format;
5786                 $devinfo->{$devname}->{storeid} = $storeid;
5787
5788                 # check permission on storage
5789                 my $pool = $opts->{pool}; # todo: do we need that?
5790                 if ($user ne 'root@pam') {
5791                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5792                 }
5793
5794                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5795             }
5796         }
5797
5798         foreach my $devname (keys %$devinfo) {
5799             die "found no device mapping information for device '$devname'\n"
5800                 if !$devinfo->{$devname}->{virtdev};
5801         }
5802
5803         my $cfg = PVE::Storage::config();
5804
5805         # create empty/temp config
5806         if ($oldconf) {
5807             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5808             foreach_drive($oldconf, sub {
5809                 my ($ds, $drive) = @_;
5810
5811                 return if drive_is_cdrom($drive);
5812
5813                 my $volid = $drive->{file};
5814
5815                 return if !$volid || $volid =~ m|^/|;
5816
5817                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5818                 return if !$path || !$owner || ($owner != $vmid);
5819
5820                 # Note: only delete disk we want to restore
5821                 # other volumes will become unused
5822                 if ($virtdev_hash->{$ds}) {
5823                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5824                     if (my $err = $@) {
5825                         warn $err;
5826                     }
5827                 }
5828             });
5829
5830             # delete vmstate files
5831             # since after the restore we have no snapshots anymore
5832             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5833                 my $snap = $oldconf->{snapshots}->{$snapname};
5834                 if ($snap->{vmstate}) {
5835                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5836                     if (my $err = $@) {
5837                         warn $err;
5838                     }
5839                 }
5840             }
5841         }
5842
5843         my $map = {};
5844         foreach my $virtdev (sort keys %$virtdev_hash) {
5845             my $d = $virtdev_hash->{$virtdev};
5846             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5847             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5848
5849             # test if requested format is supported
5850             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5851             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5852             $d->{format} = $defFormat if !$supported;
5853
5854             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5855                                                   $d->{format}, undef, $alloc_size);
5856             print STDERR "new volume ID is '$volid'\n";
5857             $d->{volid} = $volid;
5858             my $path = PVE::Storage::path($cfg, $volid);
5859
5860             PVE::Storage::activate_volumes($cfg,[$volid]);
5861
5862             my $write_zeros = 1;
5863             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5864                 $write_zeros = 0;
5865             }
5866
5867             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5868
5869             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5870             $map->{$virtdev} = $volid;
5871         }
5872
5873         $fh->seek(0, 0) || die "seek failed - $!\n";
5874
5875         my $outfd = new IO::File ($tmpfn, "w") ||
5876             die "unable to write config for VM $vmid\n";
5877
5878         my $cookie = { netcount => 0 };
5879         while (defined(my $line = <$fh>)) {
5880             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5881         }
5882
5883         $fh->close();
5884         $outfd->close();
5885     };
5886
5887     eval {
5888         # enable interrupts
5889         local $SIG{INT} =
5890             local $SIG{TERM} =
5891             local $SIG{QUIT} =
5892             local $SIG{HUP} =
5893             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5894         local $SIG{ALRM} = sub { die "got timeout\n"; };
5895
5896         $oldtimeout = alarm($timeout);
5897
5898         my $parser = sub {
5899             my $line = shift;
5900
5901             print "$line\n";
5902
5903             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5904                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5905                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5906             } elsif ($line =~ m/^CTIME: /) {
5907                 # we correctly received the vma config, so we can disable
5908                 # the timeout now for disk allocation (set to 10 minutes, so
5909                 # that we always timeout if something goes wrong)
5910                 alarm(600);
5911                 &$print_devmap();
5912                 print $fifofh "done\n";
5913                 my $tmp = $oldtimeout || 0;
5914                 $oldtimeout = undef;
5915                 alarm($tmp);
5916                 close($fifofh);
5917             }
5918         };
5919
5920         print "restore vma archive: $cmd\n";
5921         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5922     };
5923     my $err = $@;
5924
5925     alarm($oldtimeout) if $oldtimeout;
5926
5927     my $vollist = [];
5928     foreach my $devname (keys %$devinfo) {
5929         my $volid = $devinfo->{$devname}->{volid};
5930         push @$vollist, $volid if $volid;
5931     }
5932
5933     my $cfg = PVE::Storage::config();
5934     PVE::Storage::deactivate_volumes($cfg, $vollist);
5935
5936     unlink $mapfifo;
5937
5938     if ($err) {
5939         rmtree $tmpdir;
5940         unlink $tmpfn;
5941
5942         foreach my $devname (keys %$devinfo) {
5943             my $volid = $devinfo->{$devname}->{volid};
5944             next if !$volid;
5945             eval {
5946                 if ($volid =~ m|^/|) {
5947                     unlink $volid || die 'unlink failed\n';
5948                 } else {
5949                     PVE::Storage::vdisk_free($cfg, $volid);
5950                 }
5951                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5952             };
5953             print STDERR "unable to cleanup '$volid' - $@" if $@;
5954         }
5955         die $err;
5956     }
5957
5958     rmtree $tmpdir;
5959
5960     rename($tmpfn, $conffile) ||
5961         die "unable to commit configuration file '$conffile'\n";
5962
5963     PVE::Cluster::cfs_update(); # make sure we read new file
5964
5965     eval { rescan($vmid, 1); };
5966     warn $@ if $@;
5967 }
5968
5969 sub restore_tar_archive {
5970     my ($archive, $vmid, $user, $opts) = @_;
5971
5972     if ($archive ne '-') {
5973         my $firstfile = tar_archive_read_firstfile($archive);
5974         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5975             if $firstfile ne 'qemu-server.conf';
5976     }
5977
5978     my $storecfg = PVE::Storage::config();
5979
5980     # destroy existing data - keep empty config
5981     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5982     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5983
5984     my $tocmd = "/usr/lib/qemu-server/qmextract";
5985
5986     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5987     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5988     $tocmd .= ' --prealloc' if $opts->{prealloc};
5989     $tocmd .= ' --info' if $opts->{info};
5990
5991     # tar option "xf" does not autodetect compression when read from STDIN,
5992     # so we pipe to zcat
5993     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5994         PVE::Tools::shellquote("--to-command=$tocmd");
5995
5996     my $tmpdir = "/var/tmp/vzdumptmp$$";
5997     mkpath $tmpdir;
5998
5999     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6000     local $ENV{VZDUMP_VMID} = $vmid;
6001     local $ENV{VZDUMP_USER} = $user;
6002
6003     my $conffile = PVE::QemuConfig->config_file($vmid);
6004     my $tmpfn = "$conffile.$$.tmp";
6005
6006     # disable interrupts (always do cleanups)
6007     local $SIG{INT} =
6008         local $SIG{TERM} =
6009         local $SIG{QUIT} =
6010         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6011
6012     eval {
6013         # enable interrupts
6014         local $SIG{INT} =
6015             local $SIG{TERM} =
6016             local $SIG{QUIT} =
6017             local $SIG{HUP} =
6018             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6019
6020         if ($archive eq '-') {
6021             print "extracting archive from STDIN\n";
6022             run_command($cmd, input => "<&STDIN");
6023         } else {
6024             print "extracting archive '$archive'\n";
6025             run_command($cmd);
6026         }
6027
6028         return if $opts->{info};
6029
6030         # read new mapping
6031         my $map = {};
6032         my $statfile = "$tmpdir/qmrestore.stat";
6033         if (my $fd = IO::File->new($statfile, "r")) {
6034             while (defined (my $line = <$fd>)) {
6035                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6036                     $map->{$1} = $2 if $1;
6037                 } else {
6038                     print STDERR "unable to parse line in statfile - $line\n";
6039                 }
6040             }
6041             $fd->close();
6042         }
6043
6044         my $confsrc = "$tmpdir/qemu-server.conf";
6045
6046         my $srcfd = new IO::File($confsrc, "r") ||
6047             die "unable to open file '$confsrc'\n";
6048
6049         my $outfd = new IO::File ($tmpfn, "w") ||
6050             die "unable to write config for VM $vmid\n";
6051
6052         my $cookie = { netcount => 0 };
6053         while (defined (my $line = <$srcfd>)) {
6054             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6055         }
6056
6057         $srcfd->close();
6058         $outfd->close();
6059     };
6060     my $err = $@;
6061
6062     if ($err) {
6063
6064         unlink $tmpfn;
6065
6066         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6067
6068         die $err;
6069     }
6070
6071     rmtree $tmpdir;
6072
6073     rename $tmpfn, $conffile ||
6074         die "unable to commit configuration file '$conffile'\n";
6075
6076     PVE::Cluster::cfs_update(); # make sure we read new file
6077
6078     eval { rescan($vmid, 1); };
6079     warn $@ if $@;
6080 };
6081
6082 sub foreach_storage_used_by_vm {
6083     my ($conf, $func) = @_;
6084
6085     my $sidhash = {};
6086
6087     foreach_drive($conf, sub {
6088         my ($ds, $drive) = @_;
6089         return if drive_is_cdrom($drive);
6090
6091         my $volid = $drive->{file};
6092
6093         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6094         $sidhash->{$sid} = $sid if $sid;
6095     });
6096
6097     foreach my $sid (sort keys %$sidhash) {
6098         &$func($sid);
6099     }
6100 }
6101
6102 sub do_snapshots_with_qemu {
6103     my ($storecfg, $volid) = @_;
6104
6105     my $storage_name = PVE::Storage::parse_volume_id($volid);
6106
6107     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6108         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6109         return 1;
6110     }
6111
6112     if ($volid =~ m/\.(qcow2|qed)$/){
6113         return 1;
6114     }
6115
6116     return undef;
6117 }
6118
6119 sub qga_check_running {
6120     my ($vmid) = @_;
6121
6122     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6123     if ($@) {
6124         warn "Qemu Guest Agent is not running - $@";
6125         return 0;
6126     }
6127     return 1;
6128 }
6129
6130 sub template_create {
6131     my ($vmid, $conf, $disk) = @_;
6132
6133     my $storecfg = PVE::Storage::config();
6134
6135     foreach_drive($conf, sub {
6136         my ($ds, $drive) = @_;
6137
6138         return if drive_is_cdrom($drive);
6139         return if $disk && $ds ne $disk;
6140
6141         my $volid = $drive->{file};
6142         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6143
6144         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6145         $drive->{file} = $voliddst;
6146         $conf->{$ds} = print_drive($vmid, $drive);
6147         PVE::QemuConfig->write_config($vmid, $conf);
6148     });
6149 }
6150
6151 sub qemu_img_convert {
6152     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6153
6154     my $storecfg = PVE::Storage::config();
6155     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6156     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6157
6158     if ($src_storeid && $dst_storeid) {
6159
6160         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6161
6162         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6163         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6164
6165         my $src_format = qemu_img_format($src_scfg, $src_volname);
6166         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6167
6168         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6169         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6170
6171         my $cmd = [];
6172         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6173         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6174         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6175         if ($is_zero_initialized) {
6176             push @$cmd, "zeroinit:$dst_path";
6177         } else {
6178             push @$cmd, $dst_path;
6179         }
6180
6181         my $parser = sub {
6182             my $line = shift;
6183             if($line =~ m/\((\S+)\/100\%\)/){
6184                 my $percent = $1;
6185                 my $transferred = int($size * $percent / 100);
6186                 my $remaining = $size - $transferred;
6187
6188                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6189             }
6190
6191         };
6192
6193         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6194         my $err = $@;
6195         die "copy failed: $err" if $err;
6196     }
6197 }
6198
6199 sub qemu_img_format {
6200     my ($scfg, $volname) = @_;
6201
6202     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6203         return $1;
6204     } else {
6205         return "raw";
6206     }
6207 }
6208
6209 sub qemu_drive_mirror {
6210     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6211
6212     $jobs = {} if !$jobs;
6213
6214     my $qemu_target;
6215     my $format;
6216     $jobs->{"drive-$drive"} = {};
6217
6218     if ($dst_volid =~ /^nbd:/) {
6219         $qemu_target = $dst_volid;
6220         $format = "nbd";
6221     } else {
6222         my $storecfg = PVE::Storage::config();
6223         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6224
6225         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6226
6227         $format = qemu_img_format($dst_scfg, $dst_volname);
6228
6229         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6230
6231         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6232     }
6233
6234     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6235     $opts->{format} = $format if $format;
6236
6237     print "drive mirror is starting for drive-$drive\n";
6238
6239     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6240
6241     if (my $err = $@) {
6242         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6243         die "mirroring error: $err";
6244     }
6245
6246     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6247 }
6248
6249 sub qemu_drive_mirror_monitor {
6250     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6251
6252     eval {
6253         my $err_complete = 0;
6254
6255         while (1) {
6256             die "storage migration timed out\n" if $err_complete > 300;
6257
6258             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6259
6260             my $running_mirror_jobs = {};
6261             foreach my $stat (@$stats) {
6262                 next if $stat->{type} ne 'mirror';
6263                 $running_mirror_jobs->{$stat->{device}} = $stat;
6264             }
6265
6266             my $readycounter = 0;
6267
6268             foreach my $job (keys %$jobs) {
6269
6270                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6271                     print "$job : finished\n";
6272                     delete $jobs->{$job};
6273                     next;
6274                 }
6275
6276                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6277
6278                 my $busy = $running_mirror_jobs->{$job}->{busy};
6279                 my $ready = $running_mirror_jobs->{$job}->{ready};
6280                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6281                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6282                     my $remaining = $total - $transferred;
6283                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6284
6285                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6286                 }
6287
6288                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6289             }
6290
6291             last if scalar(keys %$jobs) == 0;
6292
6293             if ($readycounter == scalar(keys %$jobs)) {
6294                 print "all mirroring jobs are ready \n";
6295                 last if $skipcomplete; #do the complete later
6296
6297                 if ($vmiddst && $vmiddst != $vmid) {
6298                     my $agent_running = $qga && qga_check_running($vmid);
6299                     if ($agent_running) {
6300                         print "freeze filesystem\n";
6301                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6302                     } else {
6303                         print "suspend vm\n";
6304                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6305                     }
6306
6307                     # if we clone a disk for a new target vm, we don't switch the disk
6308                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6309
6310                     if ($agent_running) {
6311                         print "unfreeze filesystem\n";
6312                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6313                     } else {
6314                         print "resume vm\n";
6315                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6316                     }
6317
6318                     last;
6319                 } else {
6320
6321                     foreach my $job (keys %$jobs) {
6322                         # try to switch the disk if source and destination are on the same guest
6323                         print "$job: Completing block job...\n";
6324
6325                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6326                         if ($@ =~ m/cannot be completed/) {
6327                             print "$job: Block job cannot be completed, try again.\n";
6328                             $err_complete++;
6329                         }else {
6330                             print "$job: Completed successfully.\n";
6331                             $jobs->{$job}->{complete} = 1;
6332                         }
6333                     }
6334                 }
6335             }
6336             sleep 1;
6337         }
6338     };
6339     my $err = $@;
6340
6341     if ($err) {
6342         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6343         die "mirroring error: $err";
6344     }
6345
6346 }
6347
6348 sub qemu_blockjobs_cancel {
6349     my ($vmid, $jobs) = @_;
6350
6351     foreach my $job (keys %$jobs) {
6352         print "$job: Cancelling block job\n";
6353         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6354         $jobs->{$job}->{cancel} = 1;
6355     }
6356
6357     while (1) {
6358         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6359
6360         my $running_jobs = {};
6361         foreach my $stat (@$stats) {
6362             $running_jobs->{$stat->{device}} = $stat;
6363         }
6364
6365         foreach my $job (keys %$jobs) {
6366
6367             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6368                 print "$job: Done.\n";
6369                 delete $jobs->{$job};
6370             }
6371         }
6372
6373         last if scalar(keys %$jobs) == 0;
6374
6375         sleep 1;
6376     }
6377 }
6378
6379 sub clone_disk {
6380     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6381         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6382
6383     my $newvolid;
6384
6385     if (!$full) {
6386         print "create linked clone of drive $drivename ($drive->{file})\n";
6387         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6388         push @$newvollist, $newvolid;
6389     } else {
6390
6391         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6392         $storeid = $storage if $storage;
6393
6394         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6395         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6396
6397         print "create full clone of drive $drivename ($drive->{file})\n";
6398         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6399         push @$newvollist, $newvolid;
6400
6401         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6402
6403         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6404         if (!$running || $snapname) {
6405             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6406         } else {
6407
6408             my $kvmver = get_running_qemu_version ($vmid);
6409             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6410                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6411                     if $drive->{iothread};
6412             }
6413
6414             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6415         }
6416     }
6417
6418     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6419
6420     my $disk = $drive;
6421     $disk->{format} = undef;
6422     $disk->{file} = $newvolid;
6423     $disk->{size} = $size;
6424
6425     return $disk;
6426 }
6427
6428 # this only works if VM is running
6429 sub get_current_qemu_machine {
6430     my ($vmid) = @_;
6431
6432     my $cmd = { execute => 'query-machines', arguments => {} };
6433     my $res = vm_qmp_command($vmid, $cmd);
6434
6435     my ($current, $default);
6436     foreach my $e (@$res) {
6437         $default = $e->{name} if $e->{'is-default'};
6438         $current = $e->{name} if $e->{'is-current'};
6439     }
6440
6441     # fallback to the default machine if current is not supported by qemu
6442     return $current || $default || 'pc';
6443 }
6444
6445 sub get_running_qemu_version {
6446     my ($vmid) = @_;
6447     my $cmd = { execute => 'query-version', arguments => {} };
6448     my $res = vm_qmp_command($vmid, $cmd);
6449     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6450 }
6451
6452 sub qemu_machine_feature_enabled {
6453     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6454
6455     my $current_major;
6456     my $current_minor;
6457
6458     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6459
6460         $current_major = $3;
6461         $current_minor = $4;
6462
6463     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6464
6465         $current_major = $1;
6466         $current_minor = $2;
6467     }
6468
6469     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6470
6471
6472 }
6473
6474 sub qemu_machine_pxe {
6475     my ($vmid, $conf, $machine) = @_;
6476
6477     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6478
6479     foreach my $opt (keys %$conf) {
6480         next if $opt !~ m/^net(\d+)$/;
6481         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6482         next if !$net;
6483         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6484         return $machine.".pxe" if $romfile =~ m/pxe/;
6485         last;
6486     }
6487
6488     return $machine;
6489 }
6490
6491 sub qemu_use_old_bios_files {
6492     my ($machine_type) = @_;
6493
6494     return if !$machine_type;
6495
6496     my $use_old_bios_files = undef;
6497
6498     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6499         $machine_type = $1;
6500         $use_old_bios_files = 1;
6501     } else {
6502         my $kvmver = kvm_user_version();
6503         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6504         # load new efi bios files on migration. So this hack is required to allow
6505         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6506         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6507         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6508     }
6509
6510     return ($use_old_bios_files, $machine_type);
6511 }
6512
6513 sub create_efidisk {
6514     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6515
6516     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6517
6518     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6519     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6520     PVE::Storage::activate_volumes($storecfg, [$volid]);
6521
6522     my $path = PVE::Storage::path($storecfg, $volid);
6523     eval {
6524         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6525     };
6526     die "Copying EFI vars image failed: $@" if $@;
6527
6528     return ($volid, $vars_size);
6529 }
6530
6531 sub lspci {
6532
6533     my $devices = {};
6534
6535     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6536             my (undef, $id, $function) = @_;
6537             my $res = { id => $id, function => $function};
6538             push @{$devices->{$id}}, $res;
6539     });
6540
6541     # Entries should be sorted by functions.
6542     foreach my $id (keys %$devices) {
6543         my $dev = $devices->{$id};
6544         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6545     }
6546
6547     return $devices;
6548 }
6549
6550 sub vm_iothreads_list {
6551     my ($vmid) = @_;
6552
6553     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6554
6555     my $iothreads = {};
6556     foreach my $iothread (@$res) {
6557         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6558     }
6559
6560     return $iothreads;
6561 }
6562
6563 sub scsihw_infos {
6564     my ($conf, $drive) = @_;
6565
6566     my $maxdev = 0;
6567
6568     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6569         $maxdev = 7;
6570     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6571         $maxdev = 1;
6572     } else {
6573         $maxdev = 256;
6574     }
6575
6576     my $controller = int($drive->{index} / $maxdev);
6577     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6578
6579     return ($maxdev, $controller, $controller_prefix);
6580 }
6581
6582 sub add_hyperv_enlightenments {
6583     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6584
6585     return if $winversion < 6;
6586     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6587
6588     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6589
6590     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6591         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6592         push @$cpuFlags , 'hv_vapic';
6593         push @$cpuFlags , 'hv_time';
6594     } else {
6595         push @$cpuFlags , 'hv_spinlocks=0xffff';
6596     }
6597
6598     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6599         push @$cpuFlags , 'hv_reset';
6600         push @$cpuFlags , 'hv_vpindex';
6601         push @$cpuFlags , 'hv_runtime';
6602     }
6603
6604     if ($winversion >= 7) {
6605         push @$cpuFlags , 'hv_relaxed';
6606     }
6607 }
6608
6609 sub windows_version {
6610     my ($ostype) = @_;
6611
6612     return 0 if !$ostype;
6613
6614     my $winversion = 0;
6615
6616     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6617         $winversion = 5;
6618     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6619         $winversion = 6;
6620     } elsif ($ostype =~ m/^win(\d+)$/) {
6621         $winversion = $1;
6622     }
6623
6624     return $winversion;
6625 }
6626
6627 sub resolve_dst_disk_format {
6628         my ($storecfg, $storeid, $src_volname, $format) = @_;
6629         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6630
6631         if (!$format) {
6632             # if no target format is specified, use the source disk format as hint
6633             if ($src_volname) {
6634                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6635                 $format = qemu_img_format($scfg, $src_volname);
6636             } else {
6637                 return $defFormat;
6638             }
6639         }
6640
6641         # test if requested format is supported - else use default
6642         my $supported = grep { $_ eq $format } @$validFormats;
6643         $format = $defFormat if !$supported;
6644         return $format;
6645 }
6646
6647 sub resolve_first_disk {
6648     my $conf = shift;
6649     my @disks = PVE::QemuServer::valid_drive_names();
6650     my $firstdisk;
6651     foreach my $ds (reverse @disks) {
6652         next if !$conf->{$ds};
6653         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6654         next if PVE::QemuServer::drive_is_cdrom($disk);
6655         $firstdisk = $ds;
6656     }
6657     return $firstdisk;
6658 }
6659
6660 sub generate_smbios1_uuid {
6661     my ($uuid, $uuid_str);
6662     UUID::generate($uuid);
6663     UUID::unparse($uuid, $uuid_str);
6664     return "uuid=$uuid_str";
6665 }
6666
6667 # bash completion helper
6668
6669 sub complete_backup_archives {
6670     my ($cmdname, $pname, $cvalue) = @_;
6671
6672     my $cfg = PVE::Storage::config();
6673
6674     my $storeid;
6675
6676     if ($cvalue =~ m/^([^:]+):/) {
6677         $storeid = $1;
6678     }
6679
6680     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6681
6682     my $res = [];
6683     foreach my $id (keys %$data) {
6684         foreach my $item (@{$data->{$id}}) {
6685             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6686             push @$res, $item->{volid} if defined($item->{volid});
6687         }
6688     }
6689
6690     return $res;
6691 }
6692
6693 my $complete_vmid_full = sub {
6694     my ($running) = @_;
6695
6696     my $idlist = vmstatus();
6697
6698     my $res = [];
6699
6700     foreach my $id (keys %$idlist) {
6701         my $d = $idlist->{$id};
6702         if (defined($running)) {
6703             next if $d->{template};
6704             next if $running && $d->{status} ne 'running';
6705             next if !$running && $d->{status} eq 'running';
6706         }
6707         push @$res, $id;
6708
6709     }
6710     return $res;
6711 };
6712
6713 sub complete_vmid {
6714     return &$complete_vmid_full();
6715 }
6716
6717 sub complete_vmid_stopped {
6718     return &$complete_vmid_full(0);
6719 }
6720
6721 sub complete_vmid_running {
6722     return &$complete_vmid_full(1);
6723 }
6724
6725 sub complete_storage {
6726
6727     my $cfg = PVE::Storage::config();
6728     my $ids = $cfg->{ids};
6729
6730     my $res = [];
6731     foreach my $sid (keys %$ids) {
6732         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6733         next if !$ids->{$sid}->{content}->{images};
6734         push @$res, $sid;
6735     }
6736
6737     return $res;
6738 }
6739
6740 1;