PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     citype => {
 544         optional => 1,
 545         type => 'string',
 546         description => 'Specifies the cloud-init configuration format.',
 547         enum => ['configdrive2', 'nocloud'],
 548     },
 549     searchdomain => {
 550         optional => 1,
 551         type => 'string',
 552         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 553     },
 554     nameserver => {
 555         optional => 1,
 556         type => 'string', format => 'address-list',
 557         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 558     },
 559     sshkeys => {
 560         optional => 1,
 561         type => 'string',
 562         format => 'urlencoded',
 563         description => "cloud-init : Setup public SSH keys (one key per line, " .
 564                         "OpenSSH format).",
 565     },
 566     hostname => {
 567         optional => 1,
 568         description => "cloud-init: Hostname to use instead of the vm-name + search-domain.",
 569         type => 'string', format => 'dns-name',
 570         maxLength => 255,
 571     },
 572 };
 573
 574 # what about other qemu settings ?
 575 #cpu => 'string',
 576 #machine => 'string',
 577 #fda => 'file',
 578 #fdb => 'file',
 579 #mtdblock => 'file',
 580 #sd => 'file',
 581 #pflash => 'file',
 582 #snapshot => 'bool',
 583 #bootp => 'file',
 584 ##tftp => 'dir',
 585 ##smb => 'dir',
 586 #kernel => 'file',
 587 #append => 'string',
 588 #initrd => 'file',
 589 ##soundhw => 'string',
 590
 591 while (my ($k, $v) = each %$confdesc) {
 592     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 593 }
 594
 595 my $MAX_IDE_DISKS = 4;
 596 my $MAX_SCSI_DISKS = 14;
 597 my $MAX_VIRTIO_DISKS = 16;
 598 my $MAX_SATA_DISKS = 6;
 599 my $MAX_USB_DEVICES = 5;
 600 my $MAX_NETS = 32;
 601 my $MAX_UNUSED_DISKS = 8;
 602 my $MAX_HOSTPCI_DEVICES = 4;
 603 my $MAX_SERIAL_PORTS = 4;
 604 my $MAX_PARALLEL_PORTS = 3;
 605 my $MAX_NUMA = 8;
 606
 607 my $numa_fmt = {
 608     cpus => {
 609         type => "string",
 610         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 611         description => "CPUs accessing this NUMA node.",
 612         format_description => "id[-id];...",
 613     },
 614     memory => {
 615         type => "number",
 616         description => "Amount of memory this NUMA node provides.",
 617         optional => 1,
 618     },
 619     hostnodes => {
 620         type => "string",
 621         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 622         description => "Host NUMA nodes to use.",
 623         format_description => "id[-id];...",
 624         optional => 1,
 625     },
 626     policy => {
 627         type => 'string',
 628         enum => [qw(preferred bind interleave)],
 629         description => "NUMA allocation policy.",
 630         optional => 1,
 631     },
 632 };
 633 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 634 my $numadesc = {
 635     optional => 1,
 636     type => 'string', format => $numa_fmt,
 637     description => "NUMA topology.",
 638 };
 639 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 640
 641 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 642     $confdesc->{"numa$i"} = $numadesc;
 643 }
 644
 645 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 646                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 647                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 648 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 649
 650 my $net_fmt_bridge_descr = <<__EOD__;
 651 Bridge to attach the network device to. The Proxmox VE standard bridge
 652 is called 'vmbr0'.
 653
 654 If you do not specify a bridge, we create a kvm user (NATed) network
 655 device, which provides DHCP and DNS services. The following addresses
 656 are used:
 657
 658  10.0.2.2   Gateway
 659  10.0.2.3   DNS Server
 660  10.0.2.4   SMB Server
 661
 662 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 663 __EOD__
 664
 665 my $net_fmt = {
 666     macaddr => {
 667         type => 'string',
 668         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 669         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 670         format_description => "XX:XX:XX:XX:XX:XX",
 671         optional => 1,
 672     },
 673     model => {
 674         type => 'string',
 675         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 676         enum => $nic_model_list,
 677         default_key => 1,
 678     },
 679     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 680     bridge => {
 681         type => 'string',
 682         description => $net_fmt_bridge_descr,
 683         format_description => 'bridge',
 684         optional => 1,
 685     },
 686     queues => {
 687         type => 'integer',
 688         minimum => 0, maximum => 16,
 689         description => 'Number of packet queues to be used on the device.',
 690         optional => 1,
 691     },
 692     rate => {
 693         type => 'number',
 694         minimum => 0,
 695         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 696         optional => 1,
 697     },
 698     tag => {
 699         type => 'integer',
 700         minimum => 1, maximum => 4094,
 701         description => 'VLAN tag to apply to packets on this interface.',
 702         optional => 1,
 703     },
 704     trunks => {
 705         type => 'string',
 706         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 707         description => 'VLAN trunks to pass through this interface.',
 708         format_description => 'vlanid[;vlanid...]',
 709         optional => 1,
 710     },
 711     firewall => {
 712         type => 'boolean',
 713         description => 'Whether this interface should be protected by the firewall.',
 714         optional => 1,
 715     },
 716     link_down => {
 717         type => 'boolean',
 718         description => 'Whether this interface should be disconnected (like pulling the plug).',
 719         optional => 1,
 720     },
 721 };
 722
 723 my $netdesc = {
 724     optional => 1,
 725     type => 'string', format => $net_fmt,
 726     description => "Specify network devices.",
 727 };
 728
 729 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 730
 731 my $ipconfig_fmt = {
 732     ip => {
 733         type => 'string',
 734         format => 'pve-ipv4-config',
 735         format_description => 'IPv4Format/CIDR',
 736         description => 'IPv4 address in CIDR format.',
 737         optional => 1,
 738         default => 'dhcp',
 739     },
 740     gw => {
 741         type => 'string',
 742         format => 'ipv4',
 743         format_description => 'GatewayIPv4',
 744         description => 'Default gateway for IPv4 traffic.',
 745         optional => 1,
 746         requires => 'ip',
 747     },
 748     ip6 => {
 749         type => 'string',
 750         format => 'pve-ipv6-config',
 751         format_description => 'IPv6Format/CIDR',
 752         description => 'IPv6 address in CIDR format.',
 753         optional => 1,
 754         default => 'dhcp',
 755     },
 756     gw6 => {
 757         type => 'string',
 758         format => 'ipv6',
 759         format_description => 'GatewayIPv6',
 760         description => 'Default gateway for IPv6 traffic.',
 761         optional => 1,
 762         requires => 'ip6',
 763     },
 764 };
 765 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 766 my $ipconfigdesc = {
 767     optional => 1,
 768     type => 'string', format => 'pve-qm-ipconfig',
 769     description => <<'EODESCR',
 770 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 771
 772 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 773
 774 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 775 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 776
 777 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 778 EODESCR
 779 };
 780 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 781
 782 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 783     $confdesc->{"net$i"} = $netdesc;
 784     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 785 }
 786
 787 foreach my $key (keys %$confdesc_cloudinit) {
 788     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 789 }
 790
 791 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 792 sub verify_volume_id_or_qm_path {
 793     my ($volid, $noerr) = @_;
 794
 795     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 796         return $volid;
 797     }
 798
 799     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 800     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 801     if ($@) {
 802         return undef if $noerr;
 803         die $@;
 804     }
 805     return $volid;
 806 }
 807
 808 my $drivename_hash;
 809
 810 my %drivedesc_base = (
 811     volume => { alias => 'file' },
 812     file => {
 813         type => 'string',
 814         format => 'pve-volume-id-or-qm-path',
 815         default_key => 1,
 816         format_description => 'volume',
 817         description => "The drive's backing volume.",
 818     },
 819     media => {
 820         type => 'string',
 821         enum => [qw(cdrom disk)],
 822         description => "The drive's media type.",
 823         default => 'disk',
 824         optional => 1
 825     },
 826     cyls => {
 827         type => 'integer',
 828         description => "Force the drive's physical geometry to have a specific cylinder count.",
 829         optional => 1
 830     },
 831     heads => {
 832         type => 'integer',
 833         description => "Force the drive's physical geometry to have a specific head count.",
 834         optional => 1
 835     },
 836     secs => {
 837         type => 'integer',
 838         description => "Force the drive's physical geometry to have a specific sector count.",
 839         optional => 1
 840     },
 841     trans => {
 842         type => 'string',
 843         enum => [qw(none lba auto)],
 844         description => "Force disk geometry bios translation mode.",
 845         optional => 1,
 846     },
 847     snapshot => {
 848         type => 'boolean',
 849         description => "Controls qemu's snapshot mode feature."
 850             . " If activated, changes made to the disk are temporary and will"
 851             . " be discarded when the VM is shutdown.",
 852         optional => 1,
 853     },
 854     cache => {
 855         type => 'string',
 856         enum => [qw(none writethrough writeback unsafe directsync)],
 857         description => "The drive's cache mode",
 858         optional => 1,
 859     },
 860     format => get_standard_option('pve-qm-image-format'),
 861     size => {
 862         type => 'string',
 863         format => 'disk-size',
 864         format_description => 'DiskSize',
 865         description => "Disk size. This is purely informational and has no effect.",
 866         optional => 1,
 867     },
 868     backup => {
 869         type => 'boolean',
 870         description => "Whether the drive should be included when making backups.",
 871         optional => 1,
 872     },
 873     replicate => {
 874         type => 'boolean',
 875         description => 'Whether the drive should considered for replication jobs.',
 876         optional => 1,
 877         default => 1,
 878     },
 879     rerror => {
 880         type => 'string',
 881         enum => [qw(ignore report stop)],
 882         description => 'Read error action.',
 883         optional => 1,
 884     },
 885     werror => {
 886         type => 'string',
 887         enum => [qw(enospc ignore report stop)],
 888         description => 'Write error action.',
 889         optional => 1,
 890     },
 891     aio => {
 892         type => 'string',
 893         enum => [qw(native threads)],
 894         description => 'AIO type to use.',
 895         optional => 1,
 896     },
 897     discard => {
 898         type => 'string',
 899         enum => [qw(ignore on)],
 900         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 901         optional => 1,
 902     },
 903     detect_zeroes => {
 904         type => 'boolean',
 905         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 906         optional => 1,
 907     },
 908     serial => {
 909         type => 'string',
 910         format => 'urlencoded',
 911         format_description => 'serial',
 912         maxLength => 20*3, # *3 since it's %xx url enoded
 913         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 914         optional => 1,
 915     },
 916     shared => {
 917         type => 'boolean',
 918         description => 'Mark this locally-managed volume as available on all nodes',
 919         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 920         optional => 1,
 921         default => 0,
 922     }
 923 );
 924
 925 my %iothread_fmt = ( iothread => {
 926         type => 'boolean',
 927         description => "Whether to use iothreads for this drive",
 928         optional => 1,
 929 });
 930
 931 my %model_fmt = (
 932     model => {
 933         type => 'string',
 934         format => 'urlencoded',
 935         format_description => 'model',
 936         maxLength => 40*3, # *3 since it's %xx url enoded
 937         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 938         optional => 1,
 939     },
 940 );
 941
 942 my %queues_fmt = (
 943     queues => {
 944         type => 'integer',
 945         description => "Number of queues.",
 946         minimum => 2,
 947         optional => 1
 948     }
 949 );
 950
 951 my %scsiblock_fmt = (
 952     scsiblock => {
 953         type => 'boolean',
 954         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 955         optional => 1,
 956         default => 0,
 957     },
 958 );
 959
 960 my $add_throttle_desc = sub {
 961     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 962     my $d = {
 963         type => $type,
 964         format_description => $unit,
 965         description => "Maximum $what in $longunit.",
 966         optional => 1,
 967     };
 968     $d->{minimum} = $minimum if defined($minimum);
 969     $drivedesc_base{$key} = $d;
 970 };
 971 # throughput: (leaky bucket)
 972 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 973 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 974 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 975 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 976 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 977 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 978 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 979 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 980 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 981
 982 # pools: (pool of IO before throttling starts taking effect)
 983 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 984 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 985 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 986 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 987 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 988 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 989
 990 # burst lengths
 991 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 992 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 993 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 994 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 995 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 996 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 997
 998 # legacy support
 999 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1000 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1001 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1002 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1003
1004 my $ide_fmt = {
1005     %drivedesc_base,
1006     %model_fmt,
1007 };
1008 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1009
1010 my $idedesc = {
1011     optional => 1,
1012     type => 'string', format => $ide_fmt,
1013     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1014 };
1015 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1016
1017 my $scsi_fmt = {
1018     %drivedesc_base,
1019     %iothread_fmt,
1020     %queues_fmt,
1021     %scsiblock_fmt,
1022 };
1023 my $scsidesc = {
1024     optional => 1,
1025     type => 'string', format => $scsi_fmt,
1026     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1027 };
1028 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1029
1030 my $sata_fmt = {
1031     %drivedesc_base,
1032 };
1033 my $satadesc = {
1034     optional => 1,
1035     type => 'string', format => $sata_fmt,
1036     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1039
1040 my $virtio_fmt = {
1041     %drivedesc_base,
1042     %iothread_fmt,
1043 };
1044 my $virtiodesc = {
1045     optional => 1,
1046     type => 'string', format => $virtio_fmt,
1047     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1048 };
1049 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1050
1051 my $alldrive_fmt = {
1052     %drivedesc_base,
1053     %iothread_fmt,
1054     %model_fmt,
1055     %queues_fmt,
1056     %scsiblock_fmt,
1057 };
1058
1059 my $efidisk_fmt = {
1060     volume => { alias => 'file' },
1061     file => {
1062         type => 'string',
1063         format => 'pve-volume-id-or-qm-path',
1064         default_key => 1,
1065         format_description => 'volume',
1066         description => "The drive's backing volume.",
1067     },
1068     format => get_standard_option('pve-qm-image-format'),
1069     size => {
1070         type => 'string',
1071         format => 'disk-size',
1072         format_description => 'DiskSize',
1073         description => "Disk size. This is purely informational and has no effect.",
1074         optional => 1,
1075     },
1076 };
1077
1078 my $efidisk_desc = {
1079     optional => 1,
1080     type => 'string', format => $efidisk_fmt,
1081     description => "Configure a Disk for storing EFI vars",
1082 };
1083
1084 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1085
1086 my $usb_fmt = {
1087     host => {
1088         default_key => 1,
1089         type => 'string', format => 'pve-qm-usb-device',
1090         format_description => 'HOSTUSBDEVICE|spice',
1091         description => <<EODESCR,
1092 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1093
1094  'bus-port(.port)*' (decimal numbers) or
1095  'vendor_id:product_id' (hexadeciaml numbers) or
1096  'spice'
1097
1098 You can use the 'lsusb -t' command to list existing usb devices.
1099
1100 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1101
1102 The value 'spice' can be used to add a usb redirection devices for spice.
1103 EODESCR
1104     },
1105     usb3 => {
1106         optional => 1,
1107         type => 'boolean',
1108         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1109         default => 0,
1110     },
1111 };
1112
1113 my $usbdesc = {
1114     optional => 1,
1115     type => 'string', format => $usb_fmt,
1116     description => "Configure an USB device (n is 0 to 4).",
1117 };
1118 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1119
1120 # NOTE: the match-groups of this regex are used in parse_hostpci
1121 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1122 my $hostpci_fmt = {
1123     host => {
1124         default_key => 1,
1125         type => 'string',
1126         pattern => qr/$PCIRE(;$PCIRE)*/,
1127         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1128         description => <<EODESCR,
1129 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1130 of PCI virtual functions of the host. HOSTPCIID syntax is:
1131
1132 'bus:dev.func' (hexadecimal numbers)
1133
1134 You can us the 'lspci' command to list existing PCI devices.
1135 EODESCR
1136     },
1137     rombar => {
1138         type => 'boolean',
1139         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1140         optional => 1,
1141         default => 1,
1142     },
1143     romfile => {
1144         type => 'string',
1145         pattern => '[^,;]+',
1146         format_description => 'string',
1147         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1148         optional => 1,
1149     },
1150     pcie => {
1151         type => 'boolean',
1152         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1153         optional => 1,
1154         default => 0,
1155     },
1156     'x-vga' => {
1157         type => 'boolean',
1158         description =>  "Enable vfio-vga device support.",
1159         optional => 1,
1160         default => 0,
1161     },
1162 };
1163 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1164
1165 my $hostpcidesc = {
1166         optional => 1,
1167         type => 'string', format => 'pve-qm-hostpci',
1168         description => "Map host PCI devices into guest.",
1169         verbose_description =>  <<EODESCR,
1170 Map host PCI devices into guest.
1171
1172 NOTE: This option allows direct access to host hardware. So it is no longer
1173 possible to migrate such machines - use with special care.
1174
1175 CAUTION: Experimental! User reported problems with this option.
1176 EODESCR
1177 };
1178 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1179
1180 my $serialdesc = {
1181         optional => 1,
1182         type => 'string',
1183         pattern => '(/dev/.+|socket)',
1184         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1185         verbose_description =>  <<EODESCR,
1186 Create a serial device inside the VM (n is 0 to 3), and pass through a
1187 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1188 host side (use 'qm terminal' to open a terminal connection).
1189
1190 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1191
1192 CAUTION: Experimental! User reported problems with this option.
1193 EODESCR
1194 };
1195
1196 my $paralleldesc= {
1197         optional => 1,
1198         type => 'string',
1199         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1200         description =>  "Map host parallel devices (n is 0 to 2).",
1201         verbose_description =>  <<EODESCR,
1202 Map host parallel devices (n is 0 to 2).
1203
1204 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1205
1206 CAUTION: Experimental! User reported problems with this option.
1207 EODESCR
1208 };
1209
1210 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1211     $confdesc->{"parallel$i"} = $paralleldesc;
1212 }
1213
1214 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1215     $confdesc->{"serial$i"} = $serialdesc;
1216 }
1217
1218 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1219     $confdesc->{"hostpci$i"} = $hostpcidesc;
1220 }
1221
1222 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1223     $drivename_hash->{"ide$i"} = 1;
1224     $confdesc->{"ide$i"} = $idedesc;
1225 }
1226
1227 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1228     $drivename_hash->{"sata$i"} = 1;
1229     $confdesc->{"sata$i"} = $satadesc;
1230 }
1231
1232 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1233     $drivename_hash->{"scsi$i"} = 1;
1234     $confdesc->{"scsi$i"} = $scsidesc ;
1235 }
1236
1237 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1238     $drivename_hash->{"virtio$i"} = 1;
1239     $confdesc->{"virtio$i"} = $virtiodesc;
1240 }
1241
1242 $drivename_hash->{efidisk0} = 1;
1243 $confdesc->{efidisk0} = $efidisk_desc;
1244
1245 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1246     $confdesc->{"usb$i"} = $usbdesc;
1247 }
1248
1249 my $unuseddesc = {
1250     optional => 1,
1251     type => 'string', format => 'pve-volume-id',
1252     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1253 };
1254
1255 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1256     $confdesc->{"unused$i"} = $unuseddesc;
1257 }
1258
1259 my $kvm_api_version = 0;
1260
1261 sub kvm_version {
1262
1263     return $kvm_api_version if $kvm_api_version;
1264
1265     my $fh = IO::File->new("</dev/kvm") ||
1266         return 0;
1267
1268     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1269         $kvm_api_version = $v;
1270     }
1271
1272     $fh->close();
1273
1274     return  $kvm_api_version;
1275 }
1276
1277 my $kvm_user_version;
1278
1279 sub kvm_user_version {
1280
1281     return $kvm_user_version if $kvm_user_version;
1282
1283     $kvm_user_version = 'unknown';
1284
1285     my $code = sub {
1286         my $line = shift;
1287         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1288             $kvm_user_version = $2;
1289         }
1290     };
1291
1292     eval { run_command("kvm -version", outfunc => $code); };
1293     warn $@ if $@;
1294
1295     return $kvm_user_version;
1296
1297 }
1298
1299 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1300
1301 sub valid_drive_names {
1302     # order is important - used to autoselect boot disk
1303     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1304             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1305             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1306             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1307             'efidisk0');
1308 }
1309
1310 sub is_valid_drivename {
1311     my $dev = shift;
1312
1313     return defined($drivename_hash->{$dev});
1314 }
1315
1316 sub option_exists {
1317     my $key = shift;
1318     return defined($confdesc->{$key});
1319 }
1320
1321 sub nic_models {
1322     return $nic_model_list;
1323 }
1324
1325 sub os_list_description {
1326
1327     return {
1328         other => 'Other',
1329         wxp => 'Windows XP',
1330         w2k => 'Windows 2000',
1331         w2k3 =>, 'Windows 2003',
1332         w2k8 => 'Windows 2008',
1333         wvista => 'Windows Vista',
1334         win7 => 'Windows 7',
1335         win8 => 'Windows 8/2012',
1336         win10 => 'Windows 10/2016',
1337         l24 => 'Linux 2.4',
1338         l26 => 'Linux 2.6',
1339     };
1340 }
1341
1342 my $cdrom_path;
1343
1344 sub get_cdrom_path {
1345
1346     return  $cdrom_path if $cdrom_path;
1347
1348     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1349     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1350     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1351 }
1352
1353 sub get_iso_path {
1354     my ($storecfg, $vmid, $cdrom) = @_;
1355
1356     if ($cdrom eq 'cdrom') {
1357         return get_cdrom_path();
1358     } elsif ($cdrom eq 'none') {
1359         return '';
1360     } elsif ($cdrom =~ m|^/|) {
1361         return $cdrom;
1362     } else {
1363         return PVE::Storage::path($storecfg, $cdrom);
1364     }
1365 }
1366
1367 # try to convert old style file names to volume IDs
1368 sub filename_to_volume_id {
1369     my ($vmid, $file, $media) = @_;
1370
1371      if (!($file eq 'none' || $file eq 'cdrom' ||
1372           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1373
1374         return undef if $file =~ m|/|;
1375
1376         if ($media && $media eq 'cdrom') {
1377             $file = "local:iso/$file";
1378         } else {
1379             $file = "local:$vmid/$file";
1380         }
1381     }
1382
1383     return $file;
1384 }
1385
1386 sub verify_media_type {
1387     my ($opt, $vtype, $media) = @_;
1388
1389     return if !$media;
1390
1391     my $etype;
1392     if ($media eq 'disk') {
1393         $etype = 'images';
1394     } elsif ($media eq 'cdrom') {
1395         $etype = 'iso';
1396     } else {
1397         die "internal error";
1398     }
1399
1400     return if ($vtype eq $etype);
1401
1402     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1403 }
1404
1405 sub cleanup_drive_path {
1406     my ($opt, $storecfg, $drive) = @_;
1407
1408     # try to convert filesystem paths to volume IDs
1409
1410     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1411         ($drive->{file} !~ m|^/dev/.+|) &&
1412         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1413         ($drive->{file} !~ m/^\d+$/)) {
1414         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1415         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1416         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1417         verify_media_type($opt, $vtype, $drive->{media});
1418         $drive->{file} = $volid;
1419     }
1420
1421     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1422 }
1423
1424 sub parse_hotplug_features {
1425     my ($data) = @_;
1426
1427     my $res = {};
1428
1429     return $res if $data eq '0';
1430
1431     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1432
1433     foreach my $feature (PVE::Tools::split_list($data)) {
1434         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1435             $res->{$1} = 1;
1436         } else {
1437             die "invalid hotplug feature '$feature'\n";
1438         }
1439     }
1440     return $res;
1441 }
1442
1443 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1444 sub pve_verify_hotplug_features {
1445     my ($value, $noerr) = @_;
1446
1447     return $value if parse_hotplug_features($value);
1448
1449     return undef if $noerr;
1450
1451     die "unable to parse hotplug option\n";
1452 }
1453
1454 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1455 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1456 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1457 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1458 #        [,iothread=on][,serial=serial][,model=model]
1459
1460 sub parse_drive {
1461     my ($key, $data) = @_;
1462
1463     my ($interface, $index);
1464
1465     if ($key =~ m/^([^\d]+)(\d+)$/) {
1466         $interface = $1;
1467         $index = $2;
1468     } else {
1469         return undef;
1470     }
1471
1472     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1473                                      : $confdesc->{$key}->{format};
1474     if (!$desc) {
1475         warn "invalid drive key: $key\n";
1476         return undef;
1477     }
1478     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1479     return undef if !$res;
1480     $res->{interface} = $interface;
1481     $res->{index} = $index;
1482
1483     my $error = 0;
1484     foreach my $opt (qw(bps bps_rd bps_wr)) {
1485         if (my $bps = defined(delete $res->{$opt})) {
1486             if (defined($res->{"m$opt"})) {
1487                 warn "both $opt and m$opt specified\n";
1488                 ++$error;
1489                 next;
1490             }
1491             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1492         }
1493     }
1494
1495     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1496     for my $requirement (
1497         [mbps_max => 'mbps'],
1498         [mbps_rd_max => 'mbps_rd'],
1499         [mbps_wr_max => 'mbps_wr'],
1500         [miops_max => 'miops'],
1501         [miops_rd_max => 'miops_rd'],
1502         [miops_wr_max => 'miops_wr'],
1503         [bps_max_length => 'mbps_max'],
1504         [bps_rd_max_length => 'mbps_rd_max'],
1505         [bps_wr_max_length => 'mbps_wr_max'],
1506         [iops_max_length => 'iops_max'],
1507         [iops_rd_max_length => 'iops_rd_max'],
1508         [iops_wr_max_length => 'iops_wr_max']) {
1509         my ($option, $requires) = @$requirement;
1510         if ($res->{$option} && !$res->{$requires}) {
1511             warn "$option requires $requires\n";
1512             ++$error;
1513         }
1514     }
1515
1516     return undef if $error;
1517
1518     return undef if $res->{mbps_rd} && $res->{mbps};
1519     return undef if $res->{mbps_wr} && $res->{mbps};
1520     return undef if $res->{iops_rd} && $res->{iops};
1521     return undef if $res->{iops_wr} && $res->{iops};
1522
1523     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1524         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1525         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1526         return undef if $res->{interface} eq 'virtio';
1527     }
1528
1529     if (my $size = $res->{size}) {
1530         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1531     }
1532
1533     return $res;
1534 }
1535
1536 sub print_drive {
1537     my ($vmid, $drive) = @_;
1538     my $data = { %$drive };
1539     delete $data->{$_} for qw(index interface);
1540     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1541 }
1542
1543 sub scsi_inquiry {
1544     my($fh, $noerr) = @_;
1545
1546     my $SG_IO = 0x2285;
1547     my $SG_GET_VERSION_NUM = 0x2282;
1548
1549     my $versionbuf = "\x00" x 8;
1550     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1551     if (!$ret) {
1552         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1553         return undef;
1554     }
1555     my $version = unpack("I", $versionbuf);
1556     if ($version < 30000) {
1557         die "scsi generic interface too old\n"  if !$noerr;
1558         return undef;
1559     }
1560
1561     my $buf = "\x00" x 36;
1562     my $sensebuf = "\x00" x 8;
1563     my $cmd = pack("C x3 C x1", 0x12, 36);
1564
1565     # see /usr/include/scsi/sg.h
1566     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1567
1568     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1569                       length($sensebuf), 0, length($buf), $buf,
1570                       $cmd, $sensebuf, 6000);
1571
1572     $ret = ioctl($fh, $SG_IO, $packet);
1573     if (!$ret) {
1574         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1575         return undef;
1576     }
1577
1578     my @res = unpack($sg_io_hdr_t, $packet);
1579     if ($res[17] || $res[18]) {
1580         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1581         return undef;
1582     }
1583
1584     my $res = {};
1585     (my $byte0, my $byte1, $res->{vendor},
1586      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1587
1588     $res->{removable} = $byte1 & 128 ? 1 : 0;
1589     $res->{type} = $byte0 & 31;
1590
1591     return $res;
1592 }
1593
1594 sub path_is_scsi {
1595     my ($path) = @_;
1596
1597     my $fh = IO::File->new("+<$path") || return undef;
1598     my $res = scsi_inquiry($fh, 1);
1599     close($fh);
1600
1601     return $res;
1602 }
1603
1604 sub machine_type_is_q35 {
1605     my ($conf) = @_;
1606
1607     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1608 }
1609
1610 sub print_tabletdevice_full {
1611     my ($conf) = @_;
1612
1613     my $q35 = machine_type_is_q35($conf);
1614
1615     # we use uhci for old VMs because tablet driver was buggy in older qemu
1616     my $usbbus = $q35 ? "ehci" : "uhci";
1617
1618     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1619 }
1620
1621 sub print_drivedevice_full {
1622     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1623
1624     my $device = '';
1625     my $maxdev = 0;
1626
1627     if ($drive->{interface} eq 'virtio') {
1628         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1629         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1630         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1631     } elsif ($drive->{interface} eq 'scsi') {
1632
1633         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1634         my $unit = $drive->{index} % $maxdev;
1635         my $devicetype = 'hd';
1636         my $path = '';
1637         if (drive_is_cdrom($drive)) {
1638             $devicetype = 'cd';
1639         } else {
1640             if ($drive->{file} =~ m|^/|) {
1641                 $path = $drive->{file};
1642                 if (my $info = path_is_scsi($path)) {
1643                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1644                         $devicetype = 'block';
1645                     } elsif ($info->{type} == 1) { # tape
1646                         $devicetype = 'generic';
1647                     }
1648                 }
1649             } else {
1650                  $path = PVE::Storage::path($storecfg, $drive->{file});
1651             }
1652
1653             if($path =~ m/^iscsi\:\/\//){
1654                 $devicetype = 'generic';
1655             }
1656         }
1657
1658         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1659             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1660         } else {
1661             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1662         }
1663
1664     } elsif ($drive->{interface} eq 'ide'){
1665         $maxdev = 2;
1666         my $controller = int($drive->{index} / $maxdev);
1667         my $unit = $drive->{index} % $maxdev;
1668         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1669
1670         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1671         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1672             $model = URI::Escape::uri_unescape($model);
1673             $device .= ",model=$model";
1674         }
1675     } elsif ($drive->{interface} eq 'sata'){
1676         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1677         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1678         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1679     } elsif ($drive->{interface} eq 'usb') {
1680         die "implement me";
1681         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1682     } else {
1683         die "unsupported interface type";
1684     }
1685
1686     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1687
1688     return $device;
1689 }
1690
1691 sub get_initiator_name {
1692     my $initiator;
1693
1694     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1695     while (defined(my $line = <$fh>)) {
1696         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1697         $initiator = $1;
1698         last;
1699     }
1700     $fh->close();
1701
1702     return $initiator;
1703 }
1704
1705 sub print_drive_full {
1706     my ($storecfg, $vmid, $drive) = @_;
1707
1708     my $path;
1709     my $volid = $drive->{file};
1710     my $format;
1711
1712     if (drive_is_cdrom($drive)) {
1713         $path = get_iso_path($storecfg, $vmid, $volid);
1714     } else {
1715         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1716         if ($storeid) {
1717             $path = PVE::Storage::path($storecfg, $volid);
1718             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1719             $format = qemu_img_format($scfg, $volname);
1720         } else {
1721             $path = $volid;
1722             $format = "raw";
1723         }
1724    }
1725
1726     my $opts = '';
1727     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1728     foreach my $o (@qemu_drive_options) {
1729         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1730     }
1731
1732     # snapshot only accepts on|off
1733     if (defined($drive->{snapshot})) {
1734         my $v = $drive->{snapshot} ? 'on' : 'off';
1735         $opts .= ",snapshot=$v";
1736     }
1737
1738     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1739         my ($dir, $qmpname) = @$type;
1740         if (my $v = $drive->{"mbps$dir"}) {
1741             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1742         }
1743         if (my $v = $drive->{"mbps${dir}_max"}) {
1744             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1745         }
1746         if (my $v = $drive->{"bps${dir}_max_length"}) {
1747             $opts .= ",throttling.bps$qmpname-max-length=$v";
1748         }
1749         if (my $v = $drive->{"iops${dir}"}) {
1750             $opts .= ",throttling.iops$qmpname=$v";
1751         }
1752         if (my $v = $drive->{"iops${dir}_max"}) {
1753             $opts .= ",throttling.iops$qmpname-max=$v";
1754         }
1755         if (my $v = $drive->{"iops${dir}_max_length"}) {
1756             $opts .= ",throttling.iops$qmpname-max-length=$v";
1757         }
1758     }
1759
1760     if (my $serial = $drive->{serial}) {
1761         $serial = URI::Escape::uri_unescape($serial);
1762         $opts .= ",serial=$serial";
1763     }
1764
1765     $opts .= ",format=$format" if $format && !$drive->{format};
1766
1767     my $cache_direct = 0;
1768
1769     if (my $cache = $drive->{cache}) {
1770         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1771     } elsif (!drive_is_cdrom($drive)) {
1772         $opts .= ",cache=none";
1773         $cache_direct = 1;
1774     }
1775
1776     # aio native works only with O_DIRECT
1777     if (!$drive->{aio}) {
1778         if($cache_direct) {
1779             $opts .= ",aio=native";
1780         } else {
1781             $opts .= ",aio=threads";
1782         }
1783     }
1784
1785     if (!drive_is_cdrom($drive)) {
1786         my $detectzeroes;
1787         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1788             $detectzeroes = 'off';
1789         } elsif ($drive->{discard}) {
1790             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1791         } else {
1792             # This used to be our default with discard not being specified:
1793             $detectzeroes = 'on';
1794         }
1795         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1796     }
1797
1798     my $pathinfo = $path ? "file=$path," : '';
1799
1800     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1801 }
1802
1803 sub print_netdevice_full {
1804     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1805
1806     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1807
1808     my $device = $net->{model};
1809     if ($net->{model} eq 'virtio') {
1810          $device = 'virtio-net-pci';
1811      };
1812
1813     my $pciaddr = print_pci_addr("$netid", $bridges);
1814     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1815     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1816         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1817         my $vectors = $net->{queues} * 2 + 2;
1818         $tmpstr .= ",vectors=$vectors,mq=on";
1819     }
1820     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1821
1822     if ($use_old_bios_files) {
1823         my $romfile;
1824         if ($device eq 'virtio-net-pci') {
1825             $romfile = 'pxe-virtio.rom';
1826         } elsif ($device eq 'e1000') {
1827             $romfile = 'pxe-e1000.rom';
1828         } elsif ($device eq 'ne2k') {
1829             $romfile = 'pxe-ne2k_pci.rom';
1830         } elsif ($device eq 'pcnet') {
1831             $romfile = 'pxe-pcnet.rom';
1832         } elsif ($device eq 'rtl8139') {
1833             $romfile = 'pxe-rtl8139.rom';
1834         }
1835         $tmpstr .= ",romfile=$romfile" if $romfile;
1836     }
1837
1838     return $tmpstr;
1839 }
1840
1841 sub print_netdev_full {
1842     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1843
1844     my $i = '';
1845     if ($netid =~ m/^net(\d+)$/) {
1846         $i = int($1);
1847     }
1848
1849     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1850
1851     my $ifname = "tap${vmid}i$i";
1852
1853     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1854     die "interface name '$ifname' is too long (max 15 character)\n"
1855         if length($ifname) >= 16;
1856
1857     my $vhostparam = '';
1858     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1859
1860     my $vmname = $conf->{name} || "vm$vmid";
1861
1862     my $netdev = "";
1863     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1864
1865     if ($net->{bridge}) {
1866         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1867     } else {
1868         $netdev = "type=user,id=$netid,hostname=$vmname";
1869     }
1870
1871     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1872
1873     return $netdev;
1874 }
1875
1876
1877 sub print_cpu_device {
1878     my ($conf, $id) = @_;
1879
1880     my $kvm = $conf->{kvm} // 1;
1881     my $cpu = $kvm ? "kvm64" : "qemu64";
1882     if (my $cputype = $conf->{cpu}) {
1883         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1884             or die "Cannot parse cpu description: $cputype\n";
1885         $cpu = $cpuconf->{cputype};
1886     }
1887
1888     my $cores = $conf->{cores} || 1;
1889
1890     my $current_core = ($id - 1) % $cores;
1891     my $current_socket = int(($id - 1 - $current_core)/$cores);
1892
1893     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1894 }
1895
1896 sub drive_is_cloudinit {
1897     my ($drive) = @_;
1898     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1899 }
1900
1901 sub drive_is_cdrom {
1902     my ($drive, $exclude_cloudinit) = @_;
1903
1904     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1905
1906     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1907
1908 }
1909
1910 sub parse_number_sets {
1911     my ($set) = @_;
1912     my $res = [];
1913     foreach my $part (split(/;/, $set)) {
1914         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1915             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1916             push @$res, [ $1, $2 ];
1917         } else {
1918             die "invalid range: $part\n";
1919         }
1920     }
1921     return $res;
1922 }
1923
1924 sub parse_numa {
1925     my ($data) = @_;
1926
1927     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1928     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1929     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1930     return $res;
1931 }
1932
1933 sub parse_hostpci {
1934     my ($value) = @_;
1935
1936     return undef if !$value;
1937
1938     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1939
1940     my @idlist = split(/;/, $res->{host});
1941     delete $res->{host};
1942     foreach my $id (@idlist) {
1943         if ($id =~ /^$PCIRE$/) {
1944             if (defined($2)) {
1945                 push @{$res->{pciid}}, { id => $1, function => $2 };
1946             } else {
1947                 my $pcidevices = lspci($1);
1948                 $res->{pciid} = $pcidevices->{$1};
1949             }
1950         } else {
1951             # should have been caught by parse_property_string already
1952             die "failed to parse PCI id: $id\n";
1953         }
1954     }
1955     return $res;
1956 }
1957
1958 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1959 sub parse_net {
1960     my ($data) = @_;
1961
1962     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1963     if ($@) {
1964         warn $@;
1965         return undef;
1966     }
1967     if (!defined($res->{macaddr})) {
1968         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1969         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1970     }
1971     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1972     return $res;
1973 }
1974
1975 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1976 sub parse_ipconfig {
1977     my ($data) = @_;
1978
1979     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1980     if ($@) {
1981         warn $@;
1982         return undef;
1983     }
1984
1985     if ($res->{gw} && !$res->{ip}) {
1986         warn 'gateway specified without specifying an IP address';
1987         return undef;
1988     }
1989     if ($res->{gw6} && !$res->{ip6}) {
1990         warn 'IPv6 gateway specified without specifying an IPv6 address';
1991         return undef;
1992     }
1993     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1994         warn 'gateway specified together with DHCP';
1995         return undef;
1996     }
1997     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1998         # gw6 + auto/dhcp
1999         warn "IPv6 gateway specified together with $res->{ip6} address";
2000         return undef;
2001     }
2002
2003     if (!$res->{ip} && !$res->{ip6}) {
2004         return { ip => 'dhcp', ip6 => 'dhcp' };
2005     }
2006
2007     return $res;
2008 }
2009
2010 sub print_net {
2011     my $net = shift;
2012
2013     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2014 }
2015
2016 sub add_random_macs {
2017     my ($settings) = @_;
2018
2019     foreach my $opt (keys %$settings) {
2020         next if $opt !~ m/^net(\d+)$/;
2021         my $net = parse_net($settings->{$opt});
2022         next if !$net;
2023         $settings->{$opt} = print_net($net);
2024     }
2025 }
2026
2027 sub vm_is_volid_owner {
2028     my ($storecfg, $vmid, $volid) = @_;
2029
2030     if ($volid !~  m|^/|) {
2031         my ($path, $owner);
2032         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2033         if ($owner && ($owner == $vmid)) {
2034             return 1;
2035         }
2036     }
2037
2038     return undef;
2039 }
2040
2041 sub split_flagged_list {
2042     my $text = shift || '';
2043     $text =~ s/[,;]/ /g;
2044     $text =~ s/^\s+//;
2045     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2046 }
2047
2048 sub join_flagged_list {
2049     my ($how, $lst) = @_;
2050     join $how, map { $lst->{$_} . $_ } keys %$lst;
2051 }
2052
2053 sub vmconfig_delete_pending_option {
2054     my ($conf, $key, $force) = @_;
2055
2056     delete $conf->{pending}->{$key};
2057     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2058     $pending_delete_hash->{$key} = $force ? '!' : '';
2059     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2060 }
2061
2062 sub vmconfig_undelete_pending_option {
2063     my ($conf, $key) = @_;
2064
2065     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2066     delete $pending_delete_hash->{$key};
2067
2068     if (%$pending_delete_hash) {
2069         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2070     } else {
2071         delete $conf->{pending}->{delete};
2072     }
2073 }
2074
2075 sub vmconfig_register_unused_drive {
2076     my ($storecfg, $vmid, $conf, $drive) = @_;
2077
2078     if (drive_is_cloudinit($drive)) {
2079         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2080         warn $@ if $@;
2081     } elsif (!drive_is_cdrom($drive)) {
2082         my $volid = $drive->{file};
2083         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2084             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2085         }
2086     }
2087 }
2088
2089 sub vmconfig_cleanup_pending {
2090     my ($conf) = @_;
2091
2092     # remove pending changes when nothing changed
2093     my $changes;
2094     foreach my $opt (keys %{$conf->{pending}}) {
2095         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2096             $changes = 1;
2097             delete $conf->{pending}->{$opt};
2098         }
2099     }
2100
2101     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2102     my $pending_delete_hash = {};
2103     while (my ($opt, $force) = each %$current_delete_hash) {
2104         if (defined($conf->{$opt})) {
2105             $pending_delete_hash->{$opt} = $force;
2106         } else {
2107             $changes = 1;
2108         }
2109     }
2110
2111     if (%$pending_delete_hash) {
2112         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2113     } else {
2114         delete $conf->{pending}->{delete};
2115     }
2116
2117     return $changes;
2118 }
2119
2120 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2121 my $smbios1_fmt = {
2122     uuid => {
2123         type => 'string',
2124         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2125         format_description => 'UUID',
2126         description => "Set SMBIOS1 UUID.",
2127         optional => 1,
2128     },
2129     version => {
2130         type => 'string',
2131         pattern => '\S+',
2132         format_description => 'string',
2133         description => "Set SMBIOS1 version.",
2134         optional => 1,
2135     },
2136     serial => {
2137         type => 'string',
2138         pattern => '\S+',
2139         format_description => 'string',
2140         description => "Set SMBIOS1 serial number.",
2141         optional => 1,
2142     },
2143     manufacturer => {
2144         type => 'string',
2145         pattern => '\S+',
2146         format_description => 'string',
2147         description => "Set SMBIOS1 manufacturer.",
2148         optional => 1,
2149     },
2150     product => {
2151         type => 'string',
2152         pattern => '\S+',
2153         format_description => 'string',
2154         description => "Set SMBIOS1 product ID.",
2155         optional => 1,
2156     },
2157     sku => {
2158         type => 'string',
2159         pattern => '\S+',
2160         format_description => 'string',
2161         description => "Set SMBIOS1 SKU string.",
2162         optional => 1,
2163     },
2164     family => {
2165         type => 'string',
2166         pattern => '\S+',
2167         format_description => 'string',
2168         description => "Set SMBIOS1 family string.",
2169         optional => 1,
2170     },
2171 };
2172
2173 sub parse_smbios1 {
2174     my ($data) = @_;
2175
2176     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2177     warn $@ if $@;
2178     return $res;
2179 }
2180
2181 sub print_smbios1 {
2182     my ($smbios1) = @_;
2183     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2184 }
2185
2186 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2187
2188 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2189 sub verify_bootdisk {
2190     my ($value, $noerr) = @_;
2191
2192     return $value if is_valid_drivename($value);
2193
2194     return undef if $noerr;
2195
2196     die "invalid boot disk '$value'\n";
2197 }
2198
2199 sub parse_watchdog {
2200     my ($value) = @_;
2201
2202     return undef if !$value;
2203
2204     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2205     warn $@ if $@;
2206     return $res;
2207 }
2208
2209 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2210 sub verify_usb_device {
2211     my ($value, $noerr) = @_;
2212
2213     return $value if parse_usb_device($value);
2214
2215     return undef if $noerr;
2216
2217     die "unable to parse usb device\n";
2218 }
2219
2220 # add JSON properties for create and set function
2221 sub json_config_properties {
2222     my $prop = shift;
2223
2224     foreach my $opt (keys %$confdesc) {
2225         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2226         $prop->{$opt} = $confdesc->{$opt};
2227     }
2228
2229     return $prop;
2230 }
2231
2232 sub check_type {
2233     my ($key, $value) = @_;
2234
2235     die "unknown setting '$key'\n" if !$confdesc->{$key};
2236
2237     my $type = $confdesc->{$key}->{type};
2238
2239     if (!defined($value)) {
2240         die "got undefined value\n";
2241     }
2242
2243     if ($value =~ m/[\n\r]/) {
2244         die "property contains a line feed\n";
2245     }
2246
2247     if ($type eq 'boolean') {
2248         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2249         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2250         die "type check ('boolean') failed - got '$value'\n";
2251     } elsif ($type eq 'integer') {
2252         return int($1) if $value =~ m/^(\d+)$/;
2253         die "type check ('integer') failed - got '$value'\n";
2254     } elsif ($type eq 'number') {
2255         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2256         die "type check ('number') failed - got '$value'\n";
2257     } elsif ($type eq 'string') {
2258         if (my $fmt = $confdesc->{$key}->{format}) {
2259             PVE::JSONSchema::check_format($fmt, $value);
2260             return $value;
2261         }
2262         $value =~ s/^\"(.*)\"$/$1/;
2263         return $value;
2264     } else {
2265         die "internal error"
2266     }
2267 }
2268
2269 sub check_iommu_support{
2270     #fixme : need to check IOMMU support
2271     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2272
2273     my $iommu=1;
2274     return $iommu;
2275
2276 }
2277
2278 sub touch_config {
2279     my ($vmid) = @_;
2280
2281     my $conf = PVE::QemuConfig->config_file($vmid);
2282     utime undef, undef, $conf;
2283 }
2284
2285 sub destroy_vm {
2286     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2287
2288     my $conffile = PVE::QemuConfig->config_file($vmid);
2289
2290     my $conf = PVE::QemuConfig->load_config($vmid);
2291
2292     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2293
2294     if ($conf->{template}) {
2295         # check if any base image is still used by a linked clone
2296         foreach_drive($conf, sub {
2297                 my ($ds, $drive) = @_;
2298
2299                 return if drive_is_cdrom($drive);
2300
2301                 my $volid = $drive->{file};
2302
2303                 return if !$volid || $volid =~ m|^/|;
2304
2305                 die "base volume '$volid' is still in use by linked cloned\n"
2306                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2307
2308         });
2309     }
2310
2311     # only remove disks owned by this VM
2312     foreach_drive($conf, sub {
2313         my ($ds, $drive) = @_;
2314
2315         return if drive_is_cdrom($drive, 1);
2316
2317         my $volid = $drive->{file};
2318
2319         return if !$volid || $volid =~ m|^/|;
2320
2321         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2322         return if !$path || !$owner || ($owner != $vmid);
2323
2324         eval {
2325             PVE::Storage::vdisk_free($storecfg, $volid);
2326         };
2327         warn "Could not remove disk '$volid', check manually: $@" if $@;
2328
2329     });
2330
2331     if ($keep_empty_config) {
2332         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2333     } else {
2334         unlink $conffile;
2335     }
2336
2337     # also remove unused disk
2338     eval {
2339         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2340
2341         eval {
2342             PVE::Storage::foreach_volid($dl, sub {
2343                 my ($volid, $sid, $volname, $d) = @_;
2344                 PVE::Storage::vdisk_free($storecfg, $volid);
2345             });
2346         };
2347         warn $@ if $@;
2348
2349     };
2350     warn $@ if $@;
2351 }
2352
2353 sub parse_vm_config {
2354     my ($filename, $raw) = @_;
2355
2356     return undef if !defined($raw);
2357
2358     my $res = {
2359         digest => Digest::SHA::sha1_hex($raw),
2360         snapshots => {},
2361         pending => {},
2362     };
2363
2364     $filename =~ m|/qemu-server/(\d+)\.conf$|
2365         || die "got strange filename '$filename'";
2366
2367     my $vmid = $1;
2368
2369     my $conf = $res;
2370     my $descr;
2371     my $section = '';
2372
2373     my @lines = split(/\n/, $raw);
2374     foreach my $line (@lines) {
2375         next if $line =~ m/^\s*$/;
2376
2377         if ($line =~ m/^\[PENDING\]\s*$/i) {
2378             $section = 'pending';
2379             if (defined($descr)) {
2380                 $descr =~ s/\s+$//;
2381                 $conf->{description} = $descr;
2382             }
2383             $descr = undef;
2384             $conf = $res->{$section} = {};
2385             next;
2386
2387         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2388             $section = $1;
2389             if (defined($descr)) {
2390                 $descr =~ s/\s+$//;
2391                 $conf->{description} = $descr;
2392             }
2393             $descr = undef;
2394             $conf = $res->{snapshots}->{$section} = {};
2395             next;
2396         }
2397
2398         if ($line =~ m/^\#(.*)\s*$/) {
2399             $descr = '' if !defined($descr);
2400             $descr .= PVE::Tools::decode_text($1) . "\n";
2401             next;
2402         }
2403
2404         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2405             $descr = '' if !defined($descr);
2406             $descr .= PVE::Tools::decode_text($2);
2407         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2408             $conf->{snapstate} = $1;
2409         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2410             my $key = $1;
2411             my $value = $2;
2412             $conf->{$key} = $value;
2413         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2414             my $value = $1;
2415             if ($section eq 'pending') {
2416                 $conf->{delete} = $value; # we parse this later
2417             } else {
2418                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2419             }
2420         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2421             my $key = $1;
2422             my $value = $2;
2423             eval { $value = check_type($key, $value); };
2424             if ($@) {
2425                 warn "vm $vmid - unable to parse value of '$key' - $@";
2426             } else {
2427                 $key = 'ide2' if $key eq 'cdrom';
2428                 my $fmt = $confdesc->{$key}->{format};
2429                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2430                     my $v = parse_drive($key, $value);
2431                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2432                         $v->{file} = $volid;
2433                         $value = print_drive($vmid, $v);
2434                     } else {
2435                         warn "vm $vmid - unable to parse value of '$key'\n";
2436                         next;
2437                     }
2438                 }
2439
2440                 $conf->{$key} = $value;
2441             }
2442         }
2443     }
2444
2445     if (defined($descr)) {
2446         $descr =~ s/\s+$//;
2447         $conf->{description} = $descr;
2448     }
2449     delete $res->{snapstate}; # just to be sure
2450
2451     return $res;
2452 }
2453
2454 sub write_vm_config {
2455     my ($filename, $conf) = @_;
2456
2457     delete $conf->{snapstate}; # just to be sure
2458
2459     if ($conf->{cdrom}) {
2460         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2461         $conf->{ide2} = $conf->{cdrom};
2462         delete $conf->{cdrom};
2463     }
2464
2465     # we do not use 'smp' any longer
2466     if ($conf->{sockets}) {
2467         delete $conf->{smp};
2468     } elsif ($conf->{smp}) {
2469         $conf->{sockets} = $conf->{smp};
2470         delete $conf->{cores};
2471         delete $conf->{smp};
2472     }
2473
2474     my $used_volids = {};
2475
2476     my $cleanup_config = sub {
2477         my ($cref, $pending, $snapname) = @_;
2478
2479         foreach my $key (keys %$cref) {
2480             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2481                 $key eq 'snapstate' || $key eq 'pending';
2482             my $value = $cref->{$key};
2483             if ($key eq 'delete') {
2484                 die "propertry 'delete' is only allowed in [PENDING]\n"
2485                     if !$pending;
2486                 # fixme: check syntax?
2487                 next;
2488             }
2489             eval { $value = check_type($key, $value); };
2490             die "unable to parse value of '$key' - $@" if $@;
2491
2492             $cref->{$key} = $value;
2493
2494             if (!$snapname && is_valid_drivename($key)) {
2495                 my $drive = parse_drive($key, $value);
2496                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2497             }
2498         }
2499     };
2500
2501     &$cleanup_config($conf);
2502
2503     &$cleanup_config($conf->{pending}, 1);
2504
2505     foreach my $snapname (keys %{$conf->{snapshots}}) {
2506         die "internal error" if $snapname eq 'pending';
2507         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2508     }
2509
2510     # remove 'unusedX' settings if we re-add a volume
2511     foreach my $key (keys %$conf) {
2512         my $value = $conf->{$key};
2513         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2514             delete $conf->{$key};
2515         }
2516     }
2517
2518     my $generate_raw_config = sub {
2519         my ($conf, $pending) = @_;
2520
2521         my $raw = '';
2522
2523         # add description as comment to top of file
2524         if (defined(my $descr = $conf->{description})) {
2525             if ($descr) {
2526                 foreach my $cl (split(/\n/, $descr)) {
2527                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2528                 }
2529             } else {
2530                 $raw .= "#\n" if $pending;
2531             }
2532         }
2533
2534         foreach my $key (sort keys %$conf) {
2535             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2536             $raw .= "$key: $conf->{$key}\n";
2537         }
2538         return $raw;
2539     };
2540
2541     my $raw = &$generate_raw_config($conf);
2542
2543     if (scalar(keys %{$conf->{pending}})){
2544         $raw .= "\n[PENDING]\n";
2545         $raw .= &$generate_raw_config($conf->{pending}, 1);
2546     }
2547
2548     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2549         $raw .= "\n[$snapname]\n";
2550         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2551     }
2552
2553     return $raw;
2554 }
2555
2556 sub load_defaults {
2557
2558     my $res = {};
2559
2560     # we use static defaults from our JSON schema configuration
2561     foreach my $key (keys %$confdesc) {
2562         if (defined(my $default = $confdesc->{$key}->{default})) {
2563             $res->{$key} = $default;
2564         }
2565     }
2566
2567     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2568     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2569
2570     return $res;
2571 }
2572
2573 sub config_list {
2574     my $vmlist = PVE::Cluster::get_vmlist();
2575     my $res = {};
2576     return $res if !$vmlist || !$vmlist->{ids};
2577     my $ids = $vmlist->{ids};
2578
2579     foreach my $vmid (keys %$ids) {
2580         my $d = $ids->{$vmid};
2581         next if !$d->{node} || $d->{node} ne $nodename;
2582         next if !$d->{type} || $d->{type} ne 'qemu';
2583         $res->{$vmid}->{exists} = 1;
2584     }
2585     return $res;
2586 }
2587
2588 # test if VM uses local resources (to prevent migration)
2589 sub check_local_resources {
2590     my ($conf, $noerr) = @_;
2591
2592     my $loc_res = 0;
2593
2594     $loc_res = 1 if $conf->{hostusb}; # old syntax
2595     $loc_res = 1 if $conf->{hostpci}; # old syntax
2596
2597     foreach my $k (keys %$conf) {
2598         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2599         # sockets are safe: they will recreated be on the target side post-migrate
2600         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2601         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2602     }
2603
2604     die "VM uses local resources\n" if $loc_res && !$noerr;
2605
2606     return $loc_res;
2607 }
2608
2609 # check if used storages are available on all nodes (use by migrate)
2610 sub check_storage_availability {
2611     my ($storecfg, $conf, $node) = @_;
2612
2613     foreach_drive($conf, sub {
2614         my ($ds, $drive) = @_;
2615
2616         my $volid = $drive->{file};
2617         return if !$volid;
2618
2619         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2620         return if !$sid;
2621
2622         # check if storage is available on both nodes
2623         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2624         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2625    });
2626 }
2627
2628 # list nodes where all VM images are available (used by has_feature API)
2629 sub shared_nodes {
2630     my ($conf, $storecfg) = @_;
2631
2632     my $nodelist = PVE::Cluster::get_nodelist();
2633     my $nodehash = { map { $_ => 1 } @$nodelist };
2634     my $nodename = PVE::INotify::nodename();
2635
2636     foreach_drive($conf, sub {
2637         my ($ds, $drive) = @_;
2638
2639         my $volid = $drive->{file};
2640         return if !$volid;
2641
2642         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2643         if ($storeid) {
2644             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2645             if ($scfg->{disable}) {
2646                 $nodehash = {};
2647             } elsif (my $avail = $scfg->{nodes}) {
2648                 foreach my $node (keys %$nodehash) {
2649                     delete $nodehash->{$node} if !$avail->{$node};
2650                 }
2651             } elsif (!$scfg->{shared}) {
2652                 foreach my $node (keys %$nodehash) {
2653                     delete $nodehash->{$node} if $node ne $nodename
2654                 }
2655             }
2656         }
2657     });
2658
2659     return $nodehash
2660 }
2661
2662 sub check_cmdline {
2663     my ($pidfile, $pid) = @_;
2664
2665     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2666     if (defined($fh)) {
2667         my $line = <$fh>;
2668         $fh->close;
2669         return undef if !$line;
2670         my @param = split(/\0/, $line);
2671
2672         my $cmd = $param[0];
2673         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2674
2675         for (my $i = 0; $i < scalar (@param); $i++) {
2676             my $p = $param[$i];
2677             next if !$p;
2678             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2679                 my $p = $param[$i+1];
2680                 return 1 if $p && ($p eq $pidfile);
2681                 return undef;
2682             }
2683         }
2684     }
2685     return undef;
2686 }
2687
2688 sub check_running {
2689     my ($vmid, $nocheck, $node) = @_;
2690
2691     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2692
2693     die "unable to find configuration file for VM $vmid - no such machine\n"
2694         if !$nocheck && ! -f $filename;
2695
2696     my $pidfile = pidfile_name($vmid);
2697
2698     if (my $fd = IO::File->new("<$pidfile")) {
2699         my $st = stat($fd);
2700         my $line = <$fd>;
2701         close($fd);
2702
2703         my $mtime = $st->mtime;
2704         if ($mtime > time()) {
2705             warn "file '$filename' modified in future\n";
2706         }
2707
2708         if ($line =~ m/^(\d+)$/) {
2709             my $pid = $1;
2710             if (check_cmdline($pidfile, $pid)) {
2711                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2712                     return $pid;
2713                 }
2714             }
2715         }
2716     }
2717
2718     return undef;
2719 }
2720
2721 sub vzlist {
2722
2723     my $vzlist = config_list();
2724
2725     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2726
2727     while (defined(my $de = $fd->read)) {
2728         next if $de !~ m/^(\d+)\.pid$/;
2729         my $vmid = $1;
2730         next if !defined($vzlist->{$vmid});
2731         if (my $pid = check_running($vmid)) {
2732             $vzlist->{$vmid}->{pid} = $pid;
2733         }
2734     }
2735
2736     return $vzlist;
2737 }
2738
2739 sub disksize {
2740     my ($storecfg, $conf) = @_;
2741
2742     my $bootdisk = $conf->{bootdisk};
2743     return undef if !$bootdisk;
2744     return undef if !is_valid_drivename($bootdisk);
2745
2746     return undef if !$conf->{$bootdisk};
2747
2748     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2749     return undef if !defined($drive);
2750
2751     return undef if drive_is_cdrom($drive);
2752
2753     my $volid = $drive->{file};
2754     return undef if !$volid;
2755
2756     return $drive->{size};
2757 }
2758
2759 my $last_proc_pid_stat;
2760
2761 # get VM status information
2762 # This must be fast and should not block ($full == false)
2763 # We only query KVM using QMP if $full == true (this can be slow)
2764 sub vmstatus {
2765     my ($opt_vmid, $full) = @_;
2766
2767     my $res = {};
2768
2769     my $storecfg = PVE::Storage::config();
2770
2771     my $list = vzlist();
2772     my $defaults = load_defaults();
2773
2774     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2775
2776     my $cpucount = $cpuinfo->{cpus} || 1;
2777
2778     foreach my $vmid (keys %$list) {
2779         next if $opt_vmid && ($vmid ne $opt_vmid);
2780
2781         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2782         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2783
2784         my $d = {};
2785         $d->{pid} = $list->{$vmid}->{pid};
2786
2787         # fixme: better status?
2788         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2789
2790         my $size = disksize($storecfg, $conf);
2791         if (defined($size)) {
2792             $d->{disk} = 0; # no info available
2793             $d->{maxdisk} = $size;
2794         } else {
2795             $d->{disk} = 0;
2796             $d->{maxdisk} = 0;
2797         }
2798
2799         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2800             * ($conf->{cores} || $defaults->{cores});
2801         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2802         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2803
2804         $d->{name} = $conf->{name} || "VM $vmid";
2805         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2806             : $defaults->{memory}*(1024*1024);
2807
2808         if ($conf->{balloon}) {
2809             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2810             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2811                 : $defaults->{shares};
2812         }
2813
2814         $d->{uptime} = 0;
2815         $d->{cpu} = 0;
2816         $d->{mem} = 0;
2817
2818         $d->{netout} = 0;
2819         $d->{netin} = 0;
2820
2821         $d->{diskread} = 0;
2822         $d->{diskwrite} = 0;
2823
2824         $d->{template} = PVE::QemuConfig->is_template($conf);
2825
2826         $d->{serial} = 1 if conf_has_serial($conf);
2827
2828         $res->{$vmid} = $d;
2829     }
2830
2831     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2832     foreach my $dev (keys %$netdev) {
2833         next if $dev !~ m/^tap([1-9]\d*)i/;
2834         my $vmid = $1;
2835         my $d = $res->{$vmid};
2836         next if !$d;
2837
2838         $d->{netout} += $netdev->{$dev}->{receive};
2839         $d->{netin} += $netdev->{$dev}->{transmit};
2840
2841         if ($full) {
2842             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2843             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2844         }
2845
2846     }
2847
2848     my $ctime = gettimeofday;
2849
2850     foreach my $vmid (keys %$list) {
2851
2852         my $d = $res->{$vmid};
2853         my $pid = $d->{pid};
2854         next if !$pid;
2855
2856         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2857         next if !$pstat; # not running
2858
2859         my $used = $pstat->{utime} + $pstat->{stime};
2860
2861         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2862
2863         if ($pstat->{vsize}) {
2864             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2865         }
2866
2867         my $old = $last_proc_pid_stat->{$pid};
2868         if (!$old) {
2869             $last_proc_pid_stat->{$pid} = {
2870                 time => $ctime,
2871                 used => $used,
2872                 cpu => 0,
2873             };
2874             next;
2875         }
2876
2877         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2878
2879         if ($dtime > 1000) {
2880             my $dutime = $used -  $old->{used};
2881
2882             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2883             $last_proc_pid_stat->{$pid} = {
2884                 time => $ctime,
2885                 used => $used,
2886                 cpu => $d->{cpu},
2887             };
2888         } else {
2889             $d->{cpu} = $old->{cpu};
2890         }
2891     }
2892
2893     return $res if !$full;
2894
2895     my $qmpclient = PVE::QMPClient->new();
2896
2897     my $ballooncb = sub {
2898         my ($vmid, $resp) = @_;
2899
2900         my $info = $resp->{'return'};
2901         return if !$info->{max_mem};
2902
2903         my $d = $res->{$vmid};
2904
2905         # use memory assigned to VM
2906         $d->{maxmem} = $info->{max_mem};
2907         $d->{balloon} = $info->{actual};
2908
2909         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2910             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2911             $d->{freemem} = $info->{free_mem};
2912         }
2913
2914         $d->{ballooninfo} = $info;
2915     };
2916
2917     my $blockstatscb = sub {
2918         my ($vmid, $resp) = @_;
2919         my $data = $resp->{'return'} || [];
2920         my $totalrdbytes = 0;
2921         my $totalwrbytes = 0;
2922
2923         for my $blockstat (@$data) {
2924             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2925             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2926
2927             $blockstat->{device} =~ s/drive-//;
2928             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2929         }
2930         $res->{$vmid}->{diskread} = $totalrdbytes;
2931         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2932     };
2933
2934     my $statuscb = sub {
2935         my ($vmid, $resp) = @_;
2936
2937         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2938         # this fails if ballon driver is not loaded, so this must be
2939         # the last commnand (following command are aborted if this fails).
2940         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2941
2942         my $status = 'unknown';
2943         if (!defined($status = $resp->{'return'}->{status})) {
2944             warn "unable to get VM status\n";
2945             return;
2946         }
2947
2948         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2949     };
2950
2951     foreach my $vmid (keys %$list) {
2952         next if $opt_vmid && ($vmid ne $opt_vmid);
2953         next if !$res->{$vmid}->{pid}; # not running
2954         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2955     }
2956
2957     $qmpclient->queue_execute(undef, 2);
2958
2959     foreach my $vmid (keys %$list) {
2960         next if $opt_vmid && ($vmid ne $opt_vmid);
2961         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2962     }
2963
2964     return $res;
2965 }
2966
2967 sub foreach_drive {
2968     my ($conf, $func, @param) = @_;
2969
2970     foreach my $ds (valid_drive_names()) {
2971         next if !defined($conf->{$ds});
2972
2973         my $drive = parse_drive($ds, $conf->{$ds});
2974         next if !$drive;
2975
2976         &$func($ds, $drive, @param);
2977     }
2978 }
2979
2980 sub foreach_volid {
2981     my ($conf, $func, @param) = @_;
2982
2983     my $volhash = {};
2984
2985     my $test_volid = sub {
2986         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2987
2988         return if !$volid;
2989
2990         $volhash->{$volid}->{cdrom} //= 1;
2991         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2992
2993         $volhash->{$volid}->{replicate} //= 0;
2994         $volhash->{$volid}->{replicate} = 1 if $replicate;
2995
2996         $volhash->{$volid}->{shared} //= 0;
2997         $volhash->{$volid}->{shared} = 1 if $shared;
2998
2999         $volhash->{$volid}->{referenced_in_config} //= 0;
3000         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3001
3002         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3003             if defined($snapname);
3004     };
3005
3006     foreach_drive($conf, sub {
3007         my ($ds, $drive) = @_;
3008         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3009     });
3010
3011     foreach my $snapname (keys %{$conf->{snapshots}}) {
3012         my $snap = $conf->{snapshots}->{$snapname};
3013         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3014         foreach_drive($snap, sub {
3015             my ($ds, $drive) = @_;
3016             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3017         });
3018     }
3019
3020     foreach my $volid (keys %$volhash) {
3021         &$func($volid, $volhash->{$volid}, @param);
3022     }
3023 }
3024
3025 sub conf_has_serial {
3026     my ($conf) = @_;
3027
3028     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3029         if ($conf->{"serial$i"}) {
3030             return 1;
3031         }
3032     }
3033
3034     return 0;
3035 }
3036
3037 sub vga_conf_has_spice {
3038     my ($vga) = @_;
3039
3040     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3041
3042     return $1 || 1;
3043 }
3044
3045 sub config_to_command {
3046     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3047
3048     my $cmd = [];
3049     my $globalFlags = [];
3050     my $machineFlags = [];
3051     my $rtcFlags = [];
3052     my $cpuFlags = [];
3053     my $devices = [];
3054     my $pciaddr = '';
3055     my $bridges = {};
3056     my $kvmver = kvm_user_version();
3057     my $vernum = 0; # unknown
3058     my $ostype = $conf->{ostype};
3059     my $winversion = windows_version($ostype);
3060     my $kvm = $conf->{kvm} // 1;
3061
3062     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3063
3064     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3065         $vernum = $1*1000000+$2*1000;
3066     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3067         $vernum = $1*1000000+$2*1000+$3;
3068     }
3069
3070     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3071
3072     my $have_ovz = -f '/proc/vz/vestat';
3073
3074     my $q35 = machine_type_is_q35($conf);
3075     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3076     my $machine_type = $forcemachine || $conf->{machine};
3077     my $use_old_bios_files = undef;
3078     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3079
3080     my $cpuunits = defined($conf->{cpuunits}) ?
3081             $conf->{cpuunits} : $defaults->{cpuunits};
3082
3083     push @$cmd, '/usr/bin/kvm';
3084
3085     push @$cmd, '-id', $vmid;
3086
3087     my $use_virtio = 0;
3088
3089     my $qmpsocket = qmp_socket($vmid);
3090     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3091     push @$cmd, '-mon', "chardev=qmp,mode=control";
3092
3093
3094     push @$cmd, '-pidfile' , pidfile_name($vmid);
3095
3096     push @$cmd, '-daemonize';
3097
3098     if ($conf->{smbios1}) {
3099         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3100     }
3101
3102     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3103         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3104
3105         my $path;
3106         my $format;
3107         if (my $efidisk = $conf->{efidisk0}) {
3108             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3109             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3110             $format = $d->{format};
3111             if ($storeid) {
3112                 $path = PVE::Storage::path($storecfg, $d->{file});
3113                 if (!defined($format)) {
3114                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3115                     $format = qemu_img_format($scfg, $volname);
3116                 }
3117             } else {
3118                 $path = $d->{file};
3119                 die "efidisk format must be specified\n"
3120                     if !defined($format);
3121             }
3122         } else {
3123             warn "no efidisk configured! Using temporary efivars disk.\n";
3124             $path = "/tmp/$vmid-ovmf.fd";
3125             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3126             $format = 'raw';
3127         }
3128
3129         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3130         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3131     }
3132
3133
3134     # add usb controllers
3135     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3136     push @$devices, @usbcontrollers if @usbcontrollers;
3137     my $vga = $conf->{vga};
3138
3139     my $qxlnum = vga_conf_has_spice($vga);
3140     $vga = 'qxl' if $qxlnum;
3141
3142     if (!$vga) {
3143         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3144             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3145         } else {
3146             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3147         }
3148     }
3149
3150     # enable absolute mouse coordinates (needed by vnc)
3151     my $tablet;
3152     if (defined($conf->{tablet})) {
3153         $tablet = $conf->{tablet};
3154     } else {
3155         $tablet = $defaults->{tablet};
3156         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3157         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3158     }
3159
3160     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3161
3162     my $kvm_off = 0;
3163     my $gpu_passthrough;
3164
3165     # host pci devices
3166     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3167         my $d = parse_hostpci($conf->{"hostpci$i"});
3168         next if !$d;
3169
3170         my $pcie = $d->{pcie};
3171         if($pcie){
3172             die "q35 machine model is not enabled" if !$q35;
3173             $pciaddr = print_pcie_addr("hostpci$i");
3174         }else{
3175             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3176         }
3177
3178         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3179         my $romfile = $d->{romfile};
3180
3181         my $xvga = '';
3182         if ($d->{'x-vga'}) {
3183             $xvga = ',x-vga=on';
3184             $kvm_off = 1;
3185             $vga = 'none';
3186             $gpu_passthrough = 1;
3187
3188             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3189                 $xvga = "";
3190             }
3191         }
3192         my $pcidevices = $d->{pciid};
3193         my $multifunction = 1 if @$pcidevices > 1;
3194
3195         my $j=0;
3196         foreach my $pcidevice (@$pcidevices) {
3197
3198             my $id = "hostpci$i";
3199             $id .= ".$j" if $multifunction;
3200             my $addr = $pciaddr;
3201             $addr .= ".$j" if $multifunction;
3202             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3203
3204             if($j == 0){
3205                 $devicestr .= "$rombar$xvga";
3206                 $devicestr .= ",multifunction=on" if $multifunction;
3207                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3208             }
3209
3210             push @$devices, '-device', $devicestr;
3211             $j++;
3212         }
3213     }
3214
3215     # usb devices
3216     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3217     push @$devices, @usbdevices if @usbdevices;
3218     # serial devices
3219     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3220         if (my $path = $conf->{"serial$i"}) {
3221             if ($path eq 'socket') {
3222                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3223                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3224                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3225             } else {
3226                 die "no such serial device\n" if ! -c $path;
3227                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3228                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3229             }
3230         }
3231     }
3232
3233     # parallel devices
3234     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3235         if (my $path = $conf->{"parallel$i"}) {
3236             die "no such parallel device\n" if ! -c $path;
3237             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3238             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3239             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3240         }
3241     }
3242
3243     my $vmname = $conf->{name} || "vm$vmid";
3244
3245     push @$cmd, '-name', $vmname;
3246
3247     my $sockets = 1;
3248     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3249     $sockets = $conf->{sockets} if  $conf->{sockets};
3250
3251     my $cores = $conf->{cores} || 1;
3252
3253     my $maxcpus = $sockets * $cores;
3254
3255     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3256
3257     my $allowed_vcpus = $cpuinfo->{cpus};
3258
3259     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3260         if ($allowed_vcpus < $maxcpus);
3261
3262     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3263
3264         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3265         for (my $i = 2; $i <= $vcpus; $i++)  {
3266             my $cpustr = print_cpu_device($conf,$i);
3267             push @$cmd, '-device', $cpustr;
3268         }
3269
3270     } else {
3271
3272         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3273     }
3274     push @$cmd, '-nodefaults';
3275
3276     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3277
3278     my $bootindex_hash = {};
3279     my $i = 1;
3280     foreach my $o (split(//, $bootorder)) {
3281         $bootindex_hash->{$o} = $i*100;
3282         $i++;
3283     }
3284
3285     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3286
3287     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3288
3289     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3290
3291     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3292
3293     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3294         my $socket = vnc_socket($vmid);
3295         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3296     } else {
3297         push @$cmd, '-nographic';
3298     }
3299
3300     # time drift fix
3301     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3302
3303     my $useLocaltime = $conf->{localtime};
3304
3305     if ($winversion >= 5) { # windows
3306         $useLocaltime = 1 if !defined($conf->{localtime});
3307
3308         # use time drift fix when acpi is enabled
3309         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3310             $tdf = 1 if !defined($conf->{tdf});
3311         }
3312     }
3313
3314     if ($winversion >= 6) {
3315         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3316         push @$cmd, '-no-hpet';
3317     }
3318
3319     push @$rtcFlags, 'driftfix=slew' if $tdf;
3320
3321     if (!$kvm) {
3322         push @$machineFlags, 'accel=tcg';
3323     }
3324
3325     if ($machine_type) {
3326         push @$machineFlags, "type=${machine_type}";
3327     }
3328
3329     if ($conf->{startdate}) {
3330         push @$rtcFlags, "base=$conf->{startdate}";
3331     } elsif ($useLocaltime) {
3332         push @$rtcFlags, 'base=localtime';
3333     }
3334
3335     my $cpu = $kvm ? "kvm64" : "qemu64";
3336     if (my $cputype = $conf->{cpu}) {
3337         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3338             or die "Cannot parse cpu description: $cputype\n";
3339         $cpu = $cpuconf->{cputype};
3340         $kvm_off = 1 if $cpuconf->{hidden};
3341
3342         if (defined(my $flags = $cpuconf->{flags})) {
3343             push @$cpuFlags, split(";", $flags);
3344         }
3345     }
3346
3347     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3348
3349     push @$cpuFlags , '-x2apic'
3350         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3351
3352     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3353
3354     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3355
3356     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3357
3358         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3359         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3360     }
3361
3362     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3363
3364     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3365
3366     push @$cpuFlags, 'kvm=off' if $kvm_off;
3367
3368     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3369         die "internal error"; # should not happen
3370
3371     push @$cpuFlags, "vendor=${cpu_vendor}"
3372         if $cpu_vendor ne 'default';
3373
3374     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3375
3376     push @$cmd, '-cpu', $cpu;
3377
3378     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3379
3380     push @$cmd, '-S' if $conf->{freeze};
3381
3382     # set keyboard layout
3383     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3384     push @$cmd, '-k', $kb if $kb;
3385
3386     # enable sound
3387     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3388     #push @$cmd, '-soundhw', 'es1370';
3389     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3390
3391     if($conf->{agent}) {
3392         my $qgasocket = qmp_socket($vmid, 1);
3393         my $pciaddr = print_pci_addr("qga0", $bridges);
3394         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3395         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3396         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3397     }
3398
3399     my $spice_port;
3400
3401     if ($qxlnum) {
3402         if ($qxlnum > 1) {
3403             if ($winversion){
3404                 for(my $i = 1; $i < $qxlnum; $i++){
3405                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3406                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3407                 }
3408             } else {
3409                 # assume other OS works like Linux
3410                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3411                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3412             }
3413         }
3414
3415         my $pciaddr = print_pci_addr("spice", $bridges);
3416
3417         my $nodename = PVE::INotify::nodename();
3418         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3419         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3420         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3421         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3422         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3423
3424         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3425
3426         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3427         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3428         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3429     }
3430
3431     # enable balloon by default, unless explicitly disabled
3432     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3433         $pciaddr = print_pci_addr("balloon0", $bridges);
3434         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3435     }
3436
3437     if ($conf->{watchdog}) {
3438         my $wdopts = parse_watchdog($conf->{watchdog});
3439         $pciaddr = print_pci_addr("watchdog", $bridges);
3440         my $watchdog = $wdopts->{model} || 'i6300esb';
3441         push @$devices, '-device', "$watchdog$pciaddr";
3442         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3443     }
3444
3445     my $vollist = [];
3446     my $scsicontroller = {};
3447     my $ahcicontroller = {};
3448     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3449
3450     # Add iscsi initiator name if available
3451     if (my $initiator = get_initiator_name()) {
3452         push @$devices, '-iscsi', "initiator-name=$initiator";
3453     }
3454
3455     foreach_drive($conf, sub {
3456         my ($ds, $drive) = @_;
3457
3458         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3459             push @$vollist, $drive->{file};
3460         }
3461
3462         # ignore efidisk here, already added in bios/fw handling code above
3463         return if $drive->{interface} eq 'efidisk';
3464
3465         $use_virtio = 1 if $ds =~ m/^virtio/;
3466
3467         if (drive_is_cdrom ($drive)) {
3468             if ($bootindex_hash->{d}) {
3469                 $drive->{bootindex} = $bootindex_hash->{d};
3470                 $bootindex_hash->{d} += 1;
3471             }
3472         } else {
3473             if ($bootindex_hash->{c}) {
3474                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3475                 $bootindex_hash->{c} += 1;
3476             }
3477         }
3478
3479         if($drive->{interface} eq 'virtio'){
3480            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3481         }
3482
3483         if ($drive->{interface} eq 'scsi') {
3484
3485             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3486
3487             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3488             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3489
3490             my $iothread = '';
3491             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3492                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3493                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3494             } elsif ($drive->{iothread}) {
3495                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3496             }
3497
3498             my $queues = '';
3499             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3500                 $queues = ",num_queues=$drive->{queues}";
3501             }
3502
3503             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3504             $scsicontroller->{$controller}=1;
3505         }
3506
3507         if ($drive->{interface} eq 'sata') {
3508            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3509            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3510            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3511            $ahcicontroller->{$controller}=1;
3512         }
3513
3514         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3515         push @$devices, '-drive',$drive_cmd;
3516         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3517     });
3518
3519     for (my $i = 0; $i < $MAX_NETS; $i++) {
3520          next if !$conf->{"net$i"};
3521          my $d = parse_net($conf->{"net$i"});
3522          next if !$d;
3523
3524          $use_virtio = 1 if $d->{model} eq 'virtio';
3525
3526          if ($bootindex_hash->{n}) {
3527             $d->{bootindex} = $bootindex_hash->{n};
3528             $bootindex_hash->{n} += 1;
3529          }
3530
3531          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3532          push @$devices, '-netdev', $netdevfull;
3533
3534          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3535          push @$devices, '-device', $netdevicefull;
3536     }
3537
3538     if (!$q35) {
3539         # add pci bridges
3540         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3541            $bridges->{1} = 1;
3542            $bridges->{2} = 1;
3543         }
3544
3545         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3546
3547         while (my ($k, $v) = each %$bridges) {
3548             $pciaddr = print_pci_addr("pci.$k");
3549             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3550         }
3551     }
3552
3553     # add custom args
3554     if ($conf->{args}) {
3555         my $aa = PVE::Tools::split_args($conf->{args});
3556         push @$cmd, @$aa;
3557     }
3558
3559     push @$cmd, @$devices;
3560     push @$cmd, '-rtc', join(',', @$rtcFlags)
3561         if scalar(@$rtcFlags);
3562     push @$cmd, '-machine', join(',', @$machineFlags)
3563         if scalar(@$machineFlags);
3564     push @$cmd, '-global', join(',', @$globalFlags)
3565         if scalar(@$globalFlags);
3566
3567     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3568 }
3569
3570 sub vnc_socket {
3571     my ($vmid) = @_;
3572     return "${var_run_tmpdir}/$vmid.vnc";
3573 }
3574
3575 sub spice_port {
3576     my ($vmid) = @_;
3577
3578     my $res = vm_mon_cmd($vmid, 'query-spice');
3579
3580     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3581 }
3582
3583 sub qmp_socket {
3584     my ($vmid, $qga) = @_;
3585     my $sockettype = $qga ? 'qga' : 'qmp';
3586     return "${var_run_tmpdir}/$vmid.$sockettype";
3587 }
3588
3589 sub pidfile_name {
3590     my ($vmid) = @_;
3591     return "${var_run_tmpdir}/$vmid.pid";
3592 }
3593
3594 sub vm_devices_list {
3595     my ($vmid) = @_;
3596
3597     my $res = vm_mon_cmd($vmid, 'query-pci');
3598     my $devices = {};
3599     foreach my $pcibus (@$res) {
3600         foreach my $device (@{$pcibus->{devices}}) {
3601             next if !$device->{'qdev_id'};
3602             if ($device->{'pci_bridge'}) {
3603                 $devices->{$device->{'qdev_id'}} = 1;
3604                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3605                     next if !$bridge_device->{'qdev_id'};
3606                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3607                     $devices->{$device->{'qdev_id'}}++;
3608                 }
3609             } else {
3610                 $devices->{$device->{'qdev_id'}} = 1;
3611             }
3612         }
3613     }
3614
3615     my $resblock = vm_mon_cmd($vmid, 'query-block');
3616     foreach my $block (@$resblock) {
3617         if($block->{device} =~ m/^drive-(\S+)/){
3618                 $devices->{$1} = 1;
3619         }
3620     }
3621
3622     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3623     foreach my $mice (@$resmice) {
3624         if ($mice->{name} eq 'QEMU HID Tablet') {
3625             $devices->{tablet} = 1;
3626             last;
3627         }
3628     }
3629
3630     # for usb devices there is no query-usb
3631     # but we can iterate over the entries in
3632     # qom-list path=/machine/peripheral
3633     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3634     foreach my $per (@$resperipheral) {
3635         if ($per->{name} =~ m/^usb\d+$/) {
3636             $devices->{$per->{name}} = 1;
3637         }
3638     }
3639
3640     return $devices;
3641 }
3642
3643 sub vm_deviceplug {
3644     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3645
3646     my $q35 = machine_type_is_q35($conf);
3647
3648     my $devices_list = vm_devices_list($vmid);
3649     return 1 if defined($devices_list->{$deviceid});
3650
3651     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3652
3653     if ($deviceid eq 'tablet') {
3654
3655         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3656
3657     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3658
3659         die "usb hotplug currently not reliable\n";
3660         # since we can't reliably hot unplug all added usb devices
3661         # and usb passthrough disables live migration
3662         # we disable usb hotplugging for now
3663         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3664
3665     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3666
3667         qemu_iothread_add($vmid, $deviceid, $device);
3668
3669         qemu_driveadd($storecfg, $vmid, $device);
3670         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3671
3672         qemu_deviceadd($vmid, $devicefull);
3673         eval { qemu_deviceaddverify($vmid, $deviceid); };
3674         if (my $err = $@) {
3675             eval { qemu_drivedel($vmid, $deviceid); };
3676             warn $@ if $@;
3677             die $err;
3678         }
3679
3680     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3681
3682
3683         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3684         my $pciaddr = print_pci_addr($deviceid);
3685         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3686
3687         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3688
3689         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3690             qemu_iothread_add($vmid, $deviceid, $device);
3691             $devicefull .= ",iothread=iothread-$deviceid";
3692         }
3693
3694         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3695             $devicefull .= ",num_queues=$device->{queues}";
3696         }
3697
3698         qemu_deviceadd($vmid, $devicefull);
3699         qemu_deviceaddverify($vmid, $deviceid);
3700
3701     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3702
3703         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3704         qemu_driveadd($storecfg, $vmid, $device);
3705
3706         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3707         eval { qemu_deviceadd($vmid, $devicefull); };
3708         if (my $err = $@) {
3709             eval { qemu_drivedel($vmid, $deviceid); };
3710             warn $@ if $@;
3711             die $err;
3712         }
3713
3714     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3715
3716         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3717
3718         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3719         my $use_old_bios_files = undef;
3720         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3721
3722         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3723         qemu_deviceadd($vmid, $netdevicefull);
3724         eval { qemu_deviceaddverify($vmid, $deviceid); };
3725         if (my $err = $@) {
3726             eval { qemu_netdevdel($vmid, $deviceid); };
3727             warn $@ if $@;
3728             die $err;
3729         }
3730
3731     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3732
3733         my $bridgeid = $2;
3734         my $pciaddr = print_pci_addr($deviceid);
3735         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3736
3737         qemu_deviceadd($vmid, $devicefull);
3738         qemu_deviceaddverify($vmid, $deviceid);
3739
3740     } else {
3741         die "can't hotplug device '$deviceid'\n";
3742     }
3743
3744     return 1;
3745 }
3746
3747 # fixme: this should raise exceptions on error!
3748 sub vm_deviceunplug {
3749     my ($vmid, $conf, $deviceid) = @_;
3750
3751     my $devices_list = vm_devices_list($vmid);
3752     return 1 if !defined($devices_list->{$deviceid});
3753
3754     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3755
3756     if ($deviceid eq 'tablet') {
3757
3758         qemu_devicedel($vmid, $deviceid);
3759
3760     } elsif ($deviceid =~ m/^usb\d+$/) {
3761
3762         die "usb hotplug currently not reliable\n";
3763         # when unplugging usb devices this way,
3764         # there may be remaining usb controllers/hubs
3765         # so we disable it for now
3766         qemu_devicedel($vmid, $deviceid);
3767         qemu_devicedelverify($vmid, $deviceid);
3768
3769     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3770
3771         qemu_devicedel($vmid, $deviceid);
3772         qemu_devicedelverify($vmid, $deviceid);
3773         qemu_drivedel($vmid, $deviceid);
3774         qemu_iothread_del($conf, $vmid, $deviceid);
3775
3776     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3777
3778         qemu_devicedel($vmid, $deviceid);
3779         qemu_devicedelverify($vmid, $deviceid);
3780         qemu_iothread_del($conf, $vmid, $deviceid);
3781
3782     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3783
3784         qemu_devicedel($vmid, $deviceid);
3785         qemu_drivedel($vmid, $deviceid);
3786         qemu_deletescsihw($conf, $vmid, $deviceid);
3787
3788     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3789
3790         qemu_devicedel($vmid, $deviceid);
3791         qemu_devicedelverify($vmid, $deviceid);
3792         qemu_netdevdel($vmid, $deviceid);
3793
3794     } else {
3795         die "can't unplug device '$deviceid'\n";
3796     }
3797
3798     return 1;
3799 }
3800
3801 sub qemu_deviceadd {
3802     my ($vmid, $devicefull) = @_;
3803
3804     $devicefull = "driver=".$devicefull;
3805     my %options =  split(/[=,]/, $devicefull);
3806
3807     vm_mon_cmd($vmid, "device_add" , %options);
3808 }
3809
3810 sub qemu_devicedel {
3811     my ($vmid, $deviceid) = @_;
3812
3813     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3814 }
3815
3816 sub qemu_iothread_add {
3817     my($vmid, $deviceid, $device) = @_;
3818
3819     if ($device->{iothread}) {
3820         my $iothreads = vm_iothreads_list($vmid);
3821         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3822     }
3823 }
3824
3825 sub qemu_iothread_del {
3826     my($conf, $vmid, $deviceid) = @_;
3827
3828     my $device = parse_drive($deviceid, $conf->{$deviceid});
3829     if ($device->{iothread}) {
3830         my $iothreads = vm_iothreads_list($vmid);
3831         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3832     }
3833 }
3834
3835 sub qemu_objectadd {
3836     my($vmid, $objectid, $qomtype) = @_;
3837
3838     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3839
3840     return 1;
3841 }
3842
3843 sub qemu_objectdel {
3844     my($vmid, $objectid) = @_;
3845
3846     vm_mon_cmd($vmid, "object-del", id => $objectid);
3847
3848     return 1;
3849 }
3850
3851 sub qemu_driveadd {
3852     my ($storecfg, $vmid, $device) = @_;
3853
3854     my $drive = print_drive_full($storecfg, $vmid, $device);
3855     $drive =~ s/\\/\\\\/g;
3856     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3857
3858     # If the command succeeds qemu prints: "OK"
3859     return 1 if $ret =~ m/OK/s;
3860
3861     die "adding drive failed: $ret\n";
3862 }
3863
3864 sub qemu_drivedel {
3865     my($vmid, $deviceid) = @_;
3866
3867     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3868     $ret =~ s/^\s+//;
3869
3870     return 1 if $ret eq "";
3871
3872     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3873     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3874
3875     die "deleting drive $deviceid failed : $ret\n";
3876 }
3877
3878 sub qemu_deviceaddverify {
3879     my ($vmid, $deviceid) = @_;
3880
3881     for (my $i = 0; $i <= 5; $i++) {
3882          my $devices_list = vm_devices_list($vmid);
3883          return 1 if defined($devices_list->{$deviceid});
3884          sleep 1;
3885     }
3886
3887     die "error on hotplug device '$deviceid'\n";
3888 }
3889
3890
3891 sub qemu_devicedelverify {
3892     my ($vmid, $deviceid) = @_;
3893
3894     # need to verify that the device is correctly removed as device_del
3895     # is async and empty return is not reliable
3896
3897     for (my $i = 0; $i <= 5; $i++) {
3898          my $devices_list = vm_devices_list($vmid);
3899          return 1 if !defined($devices_list->{$deviceid});
3900          sleep 1;
3901     }
3902
3903     die "error on hot-unplugging device '$deviceid'\n";
3904 }
3905
3906 sub qemu_findorcreatescsihw {
3907     my ($storecfg, $conf, $vmid, $device) = @_;
3908
3909     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3910
3911     my $scsihwid="$controller_prefix$controller";
3912     my $devices_list = vm_devices_list($vmid);
3913
3914     if(!defined($devices_list->{$scsihwid})) {
3915         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3916     }
3917
3918     return 1;
3919 }
3920
3921 sub qemu_deletescsihw {
3922     my ($conf, $vmid, $opt) = @_;
3923
3924     my $device = parse_drive($opt, $conf->{$opt});
3925
3926     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3927         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3928         return 1;
3929     }
3930
3931     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3932
3933     my $devices_list = vm_devices_list($vmid);
3934     foreach my $opt (keys %{$devices_list}) {
3935         if (PVE::QemuServer::is_valid_drivename($opt)) {
3936             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3937             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3938                 return 1;
3939             }
3940         }
3941     }
3942
3943     my $scsihwid="scsihw$controller";
3944
3945     vm_deviceunplug($vmid, $conf, $scsihwid);
3946
3947     return 1;
3948 }
3949
3950 sub qemu_add_pci_bridge {
3951     my ($storecfg, $conf, $vmid, $device) = @_;
3952
3953     my $bridges = {};
3954
3955     my $bridgeid;
3956
3957     print_pci_addr($device, $bridges);
3958
3959     while (my ($k, $v) = each %$bridges) {
3960         $bridgeid = $k;
3961     }
3962     return 1 if !defined($bridgeid) || $bridgeid < 1;
3963
3964     my $bridge = "pci.$bridgeid";
3965     my $devices_list = vm_devices_list($vmid);
3966
3967     if (!defined($devices_list->{$bridge})) {
3968         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3969     }
3970
3971     return 1;
3972 }
3973
3974 sub qemu_set_link_status {
3975     my ($vmid, $device, $up) = @_;
3976
3977     vm_mon_cmd($vmid, "set_link", name => $device,
3978                up => $up ? JSON::true : JSON::false);
3979 }
3980
3981 sub qemu_netdevadd {
3982     my ($vmid, $conf, $device, $deviceid) = @_;
3983
3984     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3985     my %options =  split(/[=,]/, $netdev);
3986
3987     vm_mon_cmd($vmid, "netdev_add",  %options);
3988     return 1;
3989 }
3990
3991 sub qemu_netdevdel {
3992     my ($vmid, $deviceid) = @_;
3993
3994     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3995 }
3996
3997 sub qemu_usb_hotplug {
3998     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3999
4000     return if !$device;
4001
4002     # remove the old one first
4003     vm_deviceunplug($vmid, $conf, $deviceid);
4004
4005     # check if xhci controller is necessary and available
4006     if ($device->{usb3}) {
4007
4008         my $devicelist = vm_devices_list($vmid);
4009
4010         if (!$devicelist->{xhci}) {
4011             my $pciaddr = print_pci_addr("xhci");
4012             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4013         }
4014     }
4015     my $d = parse_usb_device($device->{host});
4016     $d->{usb3} = $device->{usb3};
4017
4018     # add the new one
4019     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4020 }
4021
4022 sub qemu_cpu_hotplug {
4023     my ($vmid, $conf, $vcpus) = @_;
4024
4025     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4026
4027     my $sockets = 1;
4028     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4029     $sockets = $conf->{sockets} if  $conf->{sockets};
4030     my $cores = $conf->{cores} || 1;
4031     my $maxcpus = $sockets * $cores;
4032
4033     $vcpus = $maxcpus if !$vcpus;
4034
4035     die "you can't add more vcpus than maxcpus\n"
4036         if $vcpus > $maxcpus;
4037
4038     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4039
4040     if ($vcpus < $currentvcpus) {
4041
4042         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4043
4044             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4045                 qemu_devicedel($vmid, "cpu$i");
4046                 my $retry = 0;
4047                 my $currentrunningvcpus = undef;
4048                 while (1) {
4049                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4050                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4051                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4052                     $retry++;
4053                     sleep 1;
4054                 }
4055                 #update conf after each succesfull cpu unplug
4056                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4057                 PVE::QemuConfig->write_config($vmid, $conf);
4058             }
4059         } else {
4060             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4061         }
4062
4063         return;
4064     }
4065
4066     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4067     die "vcpus in running vm does not match its configuration\n"
4068         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4069
4070     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4071
4072         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4073             my $cpustr = print_cpu_device($conf, $i);
4074             qemu_deviceadd($vmid, $cpustr);
4075
4076             my $retry = 0;
4077             my $currentrunningvcpus = undef;
4078             while (1) {
4079                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4080                 last if scalar(@{$currentrunningvcpus}) == $i;
4081                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4082                 sleep 1;
4083                 $retry++;
4084             }
4085             #update conf after each succesfull cpu hotplug
4086             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4087             PVE::QemuConfig->write_config($vmid, $conf);
4088         }
4089     } else {
4090
4091         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4092             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4093         }
4094     }
4095 }
4096
4097 sub qemu_block_set_io_throttle {
4098     my ($vmid, $deviceid,
4099         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4100         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4101         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4102         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4103
4104     return if !check_running($vmid) ;
4105
4106     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4107         bps => int($bps),
4108         bps_rd => int($bps_rd),
4109         bps_wr => int($bps_wr),
4110         iops => int($iops),
4111         iops_rd => int($iops_rd),
4112         iops_wr => int($iops_wr),
4113         bps_max => int($bps_max),
4114         bps_rd_max => int($bps_rd_max),
4115         bps_wr_max => int($bps_wr_max),
4116         iops_max => int($iops_max),
4117         iops_rd_max => int($iops_rd_max),
4118         iops_wr_max => int($iops_wr_max),
4119         bps_max_length => int($bps_max_length),
4120         bps_rd_max_length => int($bps_rd_max_length),
4121         bps_wr_max_length => int($bps_wr_max_length),
4122         iops_max_length => int($iops_max_length),
4123         iops_rd_max_length => int($iops_rd_max_length),
4124         iops_wr_max_length => int($iops_wr_max_length),
4125     );
4126
4127 }
4128
4129 # old code, only used to shutdown old VM after update
4130 sub __read_avail {
4131     my ($fh, $timeout) = @_;
4132
4133     my $sel = new IO::Select;
4134     $sel->add($fh);
4135
4136     my $res = '';
4137     my $buf;
4138
4139     my @ready;
4140     while (scalar (@ready = $sel->can_read($timeout))) {
4141         my $count;
4142         if ($count = $fh->sysread($buf, 8192)) {
4143             if ($buf =~ /^(.*)\(qemu\) $/s) {
4144                 $res .= $1;
4145                 last;
4146             } else {
4147                 $res .= $buf;
4148             }
4149         } else {
4150             if (!defined($count)) {
4151                 die "$!\n";
4152             }
4153             last;
4154         }
4155     }
4156
4157     die "monitor read timeout\n" if !scalar(@ready);
4158
4159     return $res;
4160 }
4161
4162 # old code, only used to shutdown old VM after update
4163 sub vm_monitor_command {
4164     my ($vmid, $cmdstr, $nocheck) = @_;
4165
4166     my $res;
4167
4168     eval {
4169         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4170
4171         my $sname = "${var_run_tmpdir}/$vmid.mon";
4172
4173         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4174             die "unable to connect to VM $vmid socket - $!\n";
4175
4176         my $timeout = 3;
4177
4178         # hack: migrate sometime blocks the monitor (when migrate_downtime
4179         # is set)
4180         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4181             $timeout = 60*60; # 1 hour
4182         }
4183
4184         # read banner;
4185         my $data = __read_avail($sock, $timeout);
4186
4187         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4188             die "got unexpected qemu monitor banner\n";
4189         }
4190
4191         my $sel = new IO::Select;
4192         $sel->add($sock);
4193
4194         if (!scalar(my @ready = $sel->can_write($timeout))) {
4195             die "monitor write error - timeout";
4196         }
4197
4198         my $fullcmd = "$cmdstr\r";
4199
4200         # syslog('info', "VM $vmid monitor command: $cmdstr");
4201
4202         my $b;
4203         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4204             die "monitor write error - $!";
4205         }
4206
4207         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4208
4209         $timeout = 20;
4210
4211         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4212             $timeout = 60*60; # 1 hour
4213         } elsif ($cmdstr =~ m/^(eject|change)/) {
4214             $timeout = 60; # note: cdrom mount command is slow
4215         }
4216         if ($res = __read_avail($sock, $timeout)) {
4217
4218             my @lines = split("\r?\n", $res);
4219
4220             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4221
4222             $res = join("\n", @lines);
4223             $res .= "\n";
4224         }
4225     };
4226
4227     my $err = $@;
4228
4229     if ($err) {
4230         syslog("err", "VM $vmid monitor command failed - $err");
4231         die $err;
4232     }
4233
4234     return $res;
4235 }
4236
4237 sub qemu_block_resize {
4238     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4239
4240     my $running = check_running($vmid);
4241
4242     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4243
4244     return if !$running;
4245
4246     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4247
4248 }
4249
4250 sub qemu_volume_snapshot {
4251     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4252
4253     my $running = check_running($vmid);
4254
4255     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4256         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4257     } else {
4258         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4259     }
4260 }
4261
4262 sub qemu_volume_snapshot_delete {
4263     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4264
4265     my $running = check_running($vmid);
4266
4267     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4268         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4269     } else {
4270         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4271     }
4272 }
4273
4274 sub set_migration_caps {
4275     my ($vmid) = @_;
4276
4277     my $cap_ref = [];
4278
4279     my $enabled_cap = {
4280         "auto-converge" => 1,
4281         "xbzrle" => 1,
4282         "x-rdma-pin-all" => 0,
4283         "zero-blocks" => 0,
4284         "compress" => 0
4285     };
4286
4287     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4288
4289     for my $supported_capability (@$supported_capabilities) {
4290         push @$cap_ref, {
4291             capability => $supported_capability->{capability},
4292             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4293         };
4294     }
4295
4296     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4297 }
4298
4299 my $fast_plug_option = {
4300     'lock' => 1,
4301     'name' => 1,
4302     'onboot' => 1,
4303     'shares' => 1,
4304     'startup' => 1,
4305     'description' => 1,
4306     'protection' => 1,
4307     'vmstatestorage' => 1,
4308 };
4309
4310 # hotplug changes in [PENDING]
4311 # $selection hash can be used to only apply specified options, for
4312 # example: { cores => 1 } (only apply changed 'cores')
4313 # $errors ref is used to return error messages
4314 sub vmconfig_hotplug_pending {
4315     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4316
4317     my $defaults = load_defaults();
4318
4319     # commit values which do not have any impact on running VM first
4320     # Note: those option cannot raise errors, we we do not care about
4321     # $selection and always apply them.
4322
4323     my $add_error = sub {
4324         my ($opt, $msg) = @_;
4325         $errors->{$opt} = "hotplug problem - $msg";
4326     };
4327
4328     my $changes = 0;
4329     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4330         if ($fast_plug_option->{$opt}) {
4331             $conf->{$opt} = $conf->{pending}->{$opt};
4332             delete $conf->{pending}->{$opt};
4333             $changes = 1;
4334         }
4335     }
4336
4337     if ($changes) {
4338         PVE::QemuConfig->write_config($vmid, $conf);
4339         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4340     }
4341
4342     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4343
4344     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4345     while (my ($opt, $force) = each %$pending_delete_hash) {
4346         next if $selection && !$selection->{$opt};
4347         eval {
4348             if ($opt eq 'hotplug') {
4349                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4350             } elsif ($opt eq 'tablet') {
4351                 die "skip\n" if !$hotplug_features->{usb};
4352                 if ($defaults->{tablet}) {
4353                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4354                 } else {
4355                     vm_deviceunplug($vmid, $conf, $opt);
4356                 }
4357             } elsif ($opt =~ m/^usb\d+/) {
4358                 die "skip\n";
4359                 # since we cannot reliably hot unplug usb devices
4360                 # we are disabling it
4361                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4362                 vm_deviceunplug($vmid, $conf, $opt);
4363             } elsif ($opt eq 'vcpus') {
4364                 die "skip\n" if !$hotplug_features->{cpu};
4365                 qemu_cpu_hotplug($vmid, $conf, undef);
4366             } elsif ($opt eq 'balloon') {
4367                 # enable balloon device is not hotpluggable
4368                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4369             } elsif ($fast_plug_option->{$opt}) {
4370                 # do nothing
4371             } elsif ($opt =~ m/^net(\d+)$/) {
4372                 die "skip\n" if !$hotplug_features->{network};
4373                 vm_deviceunplug($vmid, $conf, $opt);
4374             } elsif (is_valid_drivename($opt)) {
4375                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4376                 vm_deviceunplug($vmid, $conf, $opt);
4377                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4378             } elsif ($opt =~ m/^memory$/) {
4379                 die "skip\n" if !$hotplug_features->{memory};
4380                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4381             } elsif ($opt eq 'cpuunits') {
4382                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4383             } elsif ($opt eq 'cpulimit') {
4384                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4385             } else {
4386                 die "skip\n";
4387             }
4388         };
4389         if (my $err = $@) {
4390             &$add_error($opt, $err) if $err ne "skip\n";
4391         } else {
4392             # save new config if hotplug was successful
4393             delete $conf->{$opt};
4394             vmconfig_undelete_pending_option($conf, $opt);
4395             PVE::QemuConfig->write_config($vmid, $conf);
4396             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4397         }
4398     }
4399
4400     my $apply_pending_cloudinit;
4401     $apply_pending_cloudinit = sub {
4402         my ($key, $value) = @_;
4403         $apply_pending_cloudinit = sub {}; # once is enough
4404
4405         my @cloudinit_opts = keys %$confdesc_cloudinit;
4406         foreach my $opt (keys %{$conf->{pending}}) {
4407             next if !grep { $_ eq $opt } @cloudinit_opts;
4408             $conf->{$opt} = delete $conf->{pending}->{$opt};
4409         }
4410
4411         my $new_conf = { %$conf };
4412         $new_conf->{$key} = $value;
4413         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4414     };
4415
4416     foreach my $opt (keys %{$conf->{pending}}) {
4417         next if $selection && !$selection->{$opt};
4418         my $value = $conf->{pending}->{$opt};
4419         eval {
4420             if ($opt eq 'hotplug') {
4421                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4422             } elsif ($opt eq 'tablet') {
4423                 die "skip\n" if !$hotplug_features->{usb};
4424                 if ($value == 1) {
4425                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4426                 } elsif ($value == 0) {
4427                     vm_deviceunplug($vmid, $conf, $opt);
4428                 }
4429             } elsif ($opt =~ m/^usb\d+$/) {
4430                 die "skip\n";
4431                 # since we cannot reliably hot unplug usb devices
4432                 # we are disabling it
4433                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4434                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4435                 die "skip\n" if !$d;
4436                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4437             } elsif ($opt eq 'vcpus') {
4438                 die "skip\n" if !$hotplug_features->{cpu};
4439                 qemu_cpu_hotplug($vmid, $conf, $value);
4440             } elsif ($opt eq 'balloon') {
4441                 # enable/disable balloning device is not hotpluggable
4442                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4443                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4444                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4445
4446                 # allow manual ballooning if shares is set to zero
4447                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4448                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4449                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4450                 }
4451             } elsif ($opt =~ m/^net(\d+)$/) {
4452                 # some changes can be done without hotplug
4453                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4454                                     $vmid, $opt, $value);
4455             } elsif (is_valid_drivename($opt)) {
4456                 # some changes can be done without hotplug
4457                 my $drive = parse_drive($opt, $value);
4458                 if (drive_is_cloudinit($drive)) {
4459                     &$apply_pending_cloudinit($opt, $value);
4460                 }
4461                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4462                                      $vmid, $opt, $value, 1);
4463             } elsif ($opt =~ m/^memory$/) { #dimms
4464                 die "skip\n" if !$hotplug_features->{memory};
4465                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4466             } elsif ($opt eq 'cpuunits') {
4467                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4468             } elsif ($opt eq 'cpulimit') {
4469                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4470                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4471             } else {
4472                 die "skip\n";  # skip non-hot-pluggable options
4473             }
4474         };
4475         if (my $err = $@) {
4476             &$add_error($opt, $err) if $err ne "skip\n";
4477         } else {
4478             # save new config if hotplug was successful
4479             $conf->{$opt} = $value;
4480             delete $conf->{pending}->{$opt};
4481             PVE::QemuConfig->write_config($vmid, $conf);
4482             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4483         }
4484     }
4485 }
4486
4487 sub try_deallocate_drive {
4488     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4489
4490     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4491         my $volid = $drive->{file};
4492         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4493             my $sid = PVE::Storage::parse_volume_id($volid);
4494             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4495
4496             # check if the disk is really unused
4497             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4498                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4499             PVE::Storage::vdisk_free($storecfg, $volid);
4500             return 1;
4501         } else {
4502             # If vm is not owner of this disk remove from config
4503             return 1;
4504         }
4505     }
4506
4507     return undef;
4508 }
4509
4510 sub vmconfig_delete_or_detach_drive {
4511     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4512
4513     my $drive = parse_drive($opt, $conf->{$opt});
4514
4515     my $rpcenv = PVE::RPCEnvironment::get();
4516     my $authuser = $rpcenv->get_user();
4517
4518     if ($force) {
4519         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4520         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4521     } else {
4522         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4523     }
4524 }
4525
4526 sub vmconfig_apply_pending {
4527     my ($vmid, $conf, $storecfg) = @_;
4528
4529     # cold plug
4530
4531     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4532     while (my ($opt, $force) = each %$pending_delete_hash) {
4533         die "internal error" if $opt =~ m/^unused/;
4534         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4535         if (!defined($conf->{$opt})) {
4536             vmconfig_undelete_pending_option($conf, $opt);
4537             PVE::QemuConfig->write_config($vmid, $conf);
4538         } elsif (is_valid_drivename($opt)) {
4539             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4540             vmconfig_undelete_pending_option($conf, $opt);
4541             delete $conf->{$opt};
4542             PVE::QemuConfig->write_config($vmid, $conf);
4543         } else {
4544             vmconfig_undelete_pending_option($conf, $opt);
4545             delete $conf->{$opt};
4546             PVE::QemuConfig->write_config($vmid, $conf);
4547         }
4548     }
4549
4550     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4551
4552     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4553         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4554
4555         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4556             # skip if nothing changed
4557         } elsif (is_valid_drivename($opt)) {
4558             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4559                 if defined($conf->{$opt});
4560             $conf->{$opt} = $conf->{pending}->{$opt};
4561         } else {
4562             $conf->{$opt} = $conf->{pending}->{$opt};
4563         }
4564
4565         delete $conf->{pending}->{$opt};
4566         PVE::QemuConfig->write_config($vmid, $conf);
4567     }
4568 }
4569
4570 my $safe_num_ne = sub {
4571     my ($a, $b) = @_;
4572
4573     return 0 if !defined($a) && !defined($b);
4574     return 1 if !defined($a);
4575     return 1 if !defined($b);
4576
4577     return $a != $b;
4578 };
4579
4580 my $safe_string_ne = sub {
4581     my ($a, $b) = @_;
4582
4583     return 0 if !defined($a) && !defined($b);
4584     return 1 if !defined($a);
4585     return 1 if !defined($b);
4586
4587     return $a ne $b;
4588 };
4589
4590 sub vmconfig_update_net {
4591     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4592
4593     my $newnet = parse_net($value);
4594
4595     if ($conf->{$opt}) {
4596         my $oldnet = parse_net($conf->{$opt});
4597
4598         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4599             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4600             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4601             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4602
4603             # for non online change, we try to hot-unplug
4604             die "skip\n" if !$hotplug;
4605             vm_deviceunplug($vmid, $conf, $opt);
4606         } else {
4607
4608             die "internal error" if $opt !~ m/net(\d+)/;
4609             my $iface = "tap${vmid}i$1";
4610
4611             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4612                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4613                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4614                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4615                 PVE::Network::tap_unplug($iface);
4616                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4617             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4618                 # Rate can be applied on its own but any change above needs to
4619                 # include the rate in tap_plug since OVS resets everything.
4620                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4621             }
4622
4623             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4624                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4625             }
4626
4627             return 1;
4628         }
4629     }
4630
4631     if ($hotplug) {
4632         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4633     } else {
4634         die "skip\n";
4635     }
4636 }
4637
4638 sub vmconfig_update_disk {
4639     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4640
4641     # fixme: do we need force?
4642
4643     my $drive = parse_drive($opt, $value);
4644
4645     if ($conf->{$opt}) {
4646
4647         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4648
4649             my $media = $drive->{media} || 'disk';
4650             my $oldmedia = $old_drive->{media} || 'disk';
4651             die "unable to change media type\n" if $media ne $oldmedia;
4652
4653             if (!drive_is_cdrom($old_drive)) {
4654
4655                 if ($drive->{file} ne $old_drive->{file}) {
4656
4657                     die "skip\n" if !$hotplug;
4658
4659                     # unplug and register as unused
4660                     vm_deviceunplug($vmid, $conf, $opt);
4661                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4662
4663                 } else {
4664                     # update existing disk
4665
4666                     # skip non hotpluggable value
4667                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4668                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4669                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4670                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4671                         die "skip\n";
4672                     }
4673
4674                     # apply throttle
4675                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4676                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4677                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4678                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4679                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4680                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4681                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4682                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4683                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4684                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4685                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4686                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4687                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4688                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4689                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4690                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4691                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4692                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4693
4694                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4695                                                    ($drive->{mbps} || 0)*1024*1024,
4696                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4697                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4698                                                    $drive->{iops} || 0,
4699                                                    $drive->{iops_rd} || 0,
4700                                                    $drive->{iops_wr} || 0,
4701                                                    ($drive->{mbps_max} || 0)*1024*1024,
4702                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4703                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4704                                                    $drive->{iops_max} || 0,
4705                                                    $drive->{iops_rd_max} || 0,
4706                                                    $drive->{iops_wr_max} || 0,
4707                                                    $drive->{bps_max_length} || 1,
4708                                                    $drive->{bps_rd_max_length} || 1,
4709                                                    $drive->{bps_wr_max_length} || 1,
4710                                                    $drive->{iops_max_length} || 1,
4711                                                    $drive->{iops_rd_max_length} || 1,
4712                                                    $drive->{iops_wr_max_length} || 1);
4713
4714                     }
4715
4716                     return 1;
4717                 }
4718
4719             } else { # cdrom
4720
4721                 if ($drive->{file} eq 'none') {
4722                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4723                     if (drive_is_cloudinit($old_drive)) {
4724                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4725                     }
4726                 } else {
4727                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4728                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4729                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4730                 }
4731
4732                 return 1;
4733             }
4734         }
4735     }
4736
4737     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4738     # hotplug new disks
4739     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4740     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4741 }
4742
4743 sub vm_start {
4744     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4745         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4746
4747     PVE::QemuConfig->lock_config($vmid, sub {
4748         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4749
4750         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4751
4752         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4753
4754         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4755
4756         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4757             vmconfig_apply_pending($vmid, $conf, $storecfg);
4758             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4759         }
4760
4761         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4762
4763         my $defaults = load_defaults();
4764
4765         # set environment variable useful inside network script
4766         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4767
4768         my $local_volumes = {};
4769
4770         if ($targetstorage) {
4771             foreach_drive($conf, sub {
4772                 my ($ds, $drive) = @_;
4773
4774                 return if drive_is_cdrom($drive);
4775
4776                 my $volid = $drive->{file};
4777
4778                 return if !$volid;
4779
4780                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4781
4782                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4783                 return if $scfg->{shared};
4784                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4785             });
4786
4787             my $format = undef;
4788
4789             foreach my $opt (sort keys %$local_volumes) {
4790
4791                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4792                 my $drive = parse_drive($opt, $conf->{$opt});
4793
4794                 #if remote storage is specified, use default format
4795                 if ($targetstorage && $targetstorage ne "1") {
4796                     $storeid = $targetstorage;
4797                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4798                     $format = $defFormat;
4799                 } else {
4800                     #else we use same format than original
4801                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4802                     $format = qemu_img_format($scfg, $volid);
4803                 }
4804
4805                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4806                 my $newdrive = $drive;
4807                 $newdrive->{format} = $format;
4808                 $newdrive->{file} = $newvolid;
4809                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4810                 $local_volumes->{$opt} = $drivestr;
4811                 #pass drive to conf for command line
4812                 $conf->{$opt} = $drivestr;
4813             }
4814         }
4815
4816         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4817
4818         my $migrate_port = 0;
4819         my $migrate_uri;
4820         if ($statefile) {
4821             if ($statefile eq 'tcp') {
4822                 my $localip = "localhost";
4823                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4824                 my $nodename = PVE::INotify::nodename();
4825
4826                 if (!defined($migration_type)) {
4827                     if (defined($datacenterconf->{migration}->{type})) {
4828                         $migration_type = $datacenterconf->{migration}->{type};
4829                     } else {
4830                         $migration_type = 'secure';
4831                     }
4832                 }
4833
4834                 if ($migration_type eq 'insecure') {
4835                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4836                     if ($migrate_network_addr) {
4837                         $localip = $migrate_network_addr;
4838                     } else {
4839                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4840                     }
4841
4842                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4843                 }
4844
4845                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4846                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4847                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4848                 push @$cmd, '-incoming', $migrate_uri;
4849                 push @$cmd, '-S';
4850
4851             } elsif ($statefile eq 'unix') {
4852                 # should be default for secure migrations as a ssh TCP forward
4853                 # tunnel is not deterministic reliable ready and fails regurarly
4854                 # to set up in time, so use UNIX socket forwards
4855                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4856                 unlink $socket_addr;
4857
4858                 $migrate_uri = "unix:$socket_addr";
4859
4860                 push @$cmd, '-incoming', $migrate_uri;
4861                 push @$cmd, '-S';
4862
4863             } else {
4864                 push @$cmd, '-loadstate', $statefile;
4865             }
4866         } elsif ($paused) {
4867             push @$cmd, '-S';
4868         }
4869
4870         # host pci devices
4871         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4872           my $d = parse_hostpci($conf->{"hostpci$i"});
4873           next if !$d;
4874           my $pcidevices = $d->{pciid};
4875           foreach my $pcidevice (@$pcidevices) {
4876                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4877
4878                 my $info = pci_device_info("0000:$pciid");
4879                 die "IOMMU not present\n" if !check_iommu_support();
4880                 die "no pci device info for device '$pciid'\n" if !$info;
4881                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4882                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4883           }
4884         }
4885
4886         PVE::Storage::activate_volumes($storecfg, $vollist);
4887
4888         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4889             my $cmd = [];
4890             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4891             eval  { run_command($cmd); };
4892         }
4893
4894         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4895                                                   : $defaults->{cpuunits};
4896
4897         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4898         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4899
4900         my %properties = (
4901             Slice => 'qemu.slice',
4902             KillMode => 'none',
4903             CPUShares => $cpuunits
4904         );
4905
4906         if (my $cpulimit = $conf->{cpulimit}) {
4907             $properties{CPUQuota} = int($cpulimit * 100);
4908         }
4909         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4910
4911         if ($conf->{hugepages}) {
4912
4913             my $code = sub {
4914                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4915                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4916
4917                 PVE::QemuServer::Memory::hugepages_mount();
4918                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4919
4920                 eval  {
4921                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4922                     run_command($cmd, %run_params);
4923                 };
4924
4925                 if (my $err = $@) {
4926                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4927                     die $err;
4928                 }
4929
4930                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4931             };
4932             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4933
4934         } else {
4935             eval  {
4936                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4937                 run_command($cmd, %run_params);
4938             };
4939         }
4940
4941         if (my $err = $@) {
4942             # deactivate volumes if start fails
4943             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4944             die "start failed: $err";
4945         }
4946
4947         print "migration listens on $migrate_uri\n" if $migrate_uri;
4948
4949         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4950             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4951             warn $@ if $@;
4952         }
4953
4954         #start nbd server for storage migration
4955         if ($targetstorage) {
4956             my $nodename = PVE::INotify::nodename();
4957             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4958             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4959             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4960             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4961
4962             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4963
4964             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4965
4966             foreach my $opt (sort keys %$local_volumes) {
4967                 my $volid = $local_volumes->{$opt};
4968                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4969                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4970                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4971             }
4972         }
4973
4974         if ($migratedfrom) {
4975             eval {
4976                 set_migration_caps($vmid);
4977             };
4978             warn $@ if $@;
4979
4980             if ($spice_port) {
4981                 print "spice listens on port $spice_port\n";
4982                 if ($spice_ticket) {
4983                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4984                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4985                 }
4986             }
4987
4988         } else {
4989             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4990                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4991                     if $conf->{balloon};
4992             }
4993
4994             foreach my $opt (keys %$conf) {
4995                 next if $opt !~  m/^net\d+$/;
4996                 my $nicconf = parse_net($conf->{$opt});
4997                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4998             }
4999         }
5000
5001         vm_mon_cmd_nocheck($vmid, 'qom-set',
5002                     path => "machine/peripheral/balloon0",
5003                     property => "guest-stats-polling-interval",
5004                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5005
5006     });
5007 }
5008
5009 sub vm_mon_cmd {
5010     my ($vmid, $execute, %params) = @_;
5011
5012     my $cmd = { execute => $execute, arguments => \%params };
5013     vm_qmp_command($vmid, $cmd);
5014 }
5015
5016 sub vm_mon_cmd_nocheck {
5017     my ($vmid, $execute, %params) = @_;
5018
5019     my $cmd = { execute => $execute, arguments => \%params };
5020     vm_qmp_command($vmid, $cmd, 1);
5021 }
5022
5023 sub vm_qmp_command {
5024     my ($vmid, $cmd, $nocheck) = @_;
5025
5026     my $res;
5027
5028     my $timeout;
5029     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5030         $timeout = $cmd->{arguments}->{timeout};
5031         delete $cmd->{arguments}->{timeout};
5032     }
5033
5034     eval {
5035         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5036         my $sname = qmp_socket($vmid);
5037         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5038             my $qmpclient = PVE::QMPClient->new();
5039
5040             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5041         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
5042             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
5043                 if scalar(%{$cmd->{arguments}});
5044             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
5045         } else {
5046             die "unable to open monitor socket\n";
5047         }
5048     };
5049     if (my $err = $@) {
5050         syslog("err", "VM $vmid qmp command failed - $err");
5051         die $err;
5052     }
5053
5054     return $res;
5055 }
5056
5057 sub vm_human_monitor_command {
5058     my ($vmid, $cmdline) = @_;
5059
5060     my $res;
5061
5062     my $cmd = {
5063         execute => 'human-monitor-command',
5064         arguments => { 'command-line' => $cmdline},
5065     };
5066
5067     return vm_qmp_command($vmid, $cmd);
5068 }
5069
5070 sub vm_commandline {
5071     my ($storecfg, $vmid) = @_;
5072
5073     my $conf = PVE::QemuConfig->load_config($vmid);
5074
5075     my $defaults = load_defaults();
5076
5077     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5078
5079     return PVE::Tools::cmd2string($cmd);
5080 }
5081
5082 sub vm_reset {
5083     my ($vmid, $skiplock) = @_;
5084
5085     PVE::QemuConfig->lock_config($vmid, sub {
5086
5087         my $conf = PVE::QemuConfig->load_config($vmid);
5088
5089         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5090
5091         vm_mon_cmd($vmid, "system_reset");
5092     });
5093 }
5094
5095 sub get_vm_volumes {
5096     my ($conf) = @_;
5097
5098     my $vollist = [];
5099     foreach_volid($conf, sub {
5100         my ($volid, $attr) = @_;
5101
5102         return if $volid =~ m|^/|;
5103
5104         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5105         return if !$sid;
5106
5107         push @$vollist, $volid;
5108     });
5109
5110     return $vollist;
5111 }
5112
5113 sub vm_stop_cleanup {
5114     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5115
5116     eval {
5117
5118         if (!$keepActive) {
5119             my $vollist = get_vm_volumes($conf);
5120             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5121         }
5122
5123         foreach my $ext (qw(mon qmp pid vnc qga)) {
5124             unlink "/var/run/qemu-server/${vmid}.$ext";
5125         }
5126
5127         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5128     };
5129     warn $@ if $@; # avoid errors - just warn
5130 }
5131
5132 # Note: use $nockeck to skip tests if VM configuration file exists.
5133 # We need that when migration VMs to other nodes (files already moved)
5134 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5135 sub vm_stop {
5136     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5137
5138     $force = 1 if !defined($force) && !$shutdown;
5139
5140     if ($migratedfrom){
5141         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5142         kill 15, $pid if $pid;
5143         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5144         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5145         return;
5146     }
5147
5148     PVE::QemuConfig->lock_config($vmid, sub {
5149
5150         my $pid = check_running($vmid, $nocheck);
5151         return if !$pid;
5152
5153         my $conf;
5154         if (!$nocheck) {
5155             $conf = PVE::QemuConfig->load_config($vmid);
5156             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5157             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5158                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5159                 $timeout = $opts->{down} if $opts->{down};
5160             }
5161         }
5162
5163         $timeout = 60 if !defined($timeout);
5164
5165         eval {
5166             if ($shutdown) {
5167                 if (defined($conf) && $conf->{agent}) {
5168                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5169                 } else {
5170                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5171                 }
5172             } else {
5173                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5174             }
5175         };
5176         my $err = $@;
5177
5178         if (!$err) {
5179             my $count = 0;
5180             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5181                 $count++;
5182                 sleep 1;
5183             }
5184
5185             if ($count >= $timeout) {
5186                 if ($force) {
5187                     warn "VM still running - terminating now with SIGTERM\n";
5188                     kill 15, $pid;
5189                 } else {
5190                     die "VM quit/powerdown failed - got timeout\n";
5191                 }
5192             } else {
5193                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5194                 return;
5195             }
5196         } else {
5197             if ($force) {
5198                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5199                 kill 15, $pid;
5200             } else {
5201                 die "VM quit/powerdown failed\n";
5202             }
5203         }
5204
5205         # wait again
5206         $timeout = 10;
5207
5208         my $count = 0;
5209         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5210             $count++;
5211             sleep 1;
5212         }
5213
5214         if ($count >= $timeout) {
5215             warn "VM still running - terminating now with SIGKILL\n";
5216             kill 9, $pid;
5217             sleep 1;
5218         }
5219
5220         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5221    });
5222 }
5223
5224 sub vm_suspend {
5225     my ($vmid, $skiplock) = @_;
5226
5227     PVE::QemuConfig->lock_config($vmid, sub {
5228
5229         my $conf = PVE::QemuConfig->load_config($vmid);
5230
5231         PVE::QemuConfig->check_lock($conf)
5232             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5233
5234         vm_mon_cmd($vmid, "stop");
5235     });
5236 }
5237
5238 sub vm_resume {
5239     my ($vmid, $skiplock, $nocheck) = @_;
5240
5241     PVE::QemuConfig->lock_config($vmid, sub {
5242
5243         if (!$nocheck) {
5244
5245             my $conf = PVE::QemuConfig->load_config($vmid);
5246
5247             PVE::QemuConfig->check_lock($conf)
5248                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5249
5250             vm_mon_cmd($vmid, "cont");
5251
5252         } else {
5253             vm_mon_cmd_nocheck($vmid, "cont");
5254         }
5255     });
5256 }
5257
5258 sub vm_sendkey {
5259     my ($vmid, $skiplock, $key) = @_;
5260
5261     PVE::QemuConfig->lock_config($vmid, sub {
5262
5263         my $conf = PVE::QemuConfig->load_config($vmid);
5264
5265         # there is no qmp command, so we use the human monitor command
5266         vm_human_monitor_command($vmid, "sendkey $key");
5267     });
5268 }
5269
5270 sub vm_destroy {
5271     my ($storecfg, $vmid, $skiplock) = @_;
5272
5273     PVE::QemuConfig->lock_config($vmid, sub {
5274
5275         my $conf = PVE::QemuConfig->load_config($vmid);
5276
5277         if (!check_running($vmid)) {
5278             destroy_vm($storecfg, $vmid, undef, $skiplock);
5279         } else {
5280             die "VM $vmid is running - destroy failed\n";
5281         }
5282     });
5283 }
5284
5285 # pci helpers
5286
5287 sub file_write {
5288     my ($filename, $buf) = @_;
5289
5290     my $fh = IO::File->new($filename, "w");
5291     return undef if !$fh;
5292
5293     my $res = print $fh $buf;
5294
5295     $fh->close();
5296
5297     return $res;
5298 }
5299
5300 sub pci_device_info {
5301     my ($name) = @_;
5302
5303     my $res;
5304
5305     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5306     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5307
5308     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5309     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5310
5311     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5312     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5313
5314     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5315     return undef if !defined($product) || $product !~ s/^0x//;
5316
5317     $res = {
5318         name => $name,
5319         vendor => $vendor,
5320         product => $product,
5321         domain => $domain,
5322         bus => $bus,
5323         slot => $slot,
5324         func => $func,
5325         irq => $irq,
5326         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5327     };
5328
5329     return $res;
5330 }
5331
5332 sub pci_dev_reset {
5333     my ($dev) = @_;
5334
5335     my $name = $dev->{name};
5336
5337     my $fn = "$pcisysfs/devices/$name/reset";
5338
5339     return file_write($fn, "1");
5340 }
5341
5342 sub pci_dev_bind_to_vfio {
5343     my ($dev) = @_;
5344
5345     my $name = $dev->{name};
5346
5347     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5348
5349     if (!-d $vfio_basedir) {
5350         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5351     }
5352     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5353
5354     my $testdir = "$vfio_basedir/$name";
5355     return 1 if -d $testdir;
5356
5357     my $data = "$dev->{vendor} $dev->{product}";
5358     return undef if !file_write("$vfio_basedir/new_id", $data);
5359
5360     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5361     if (!file_write($fn, $name)) {
5362         return undef if -f $fn;
5363     }
5364
5365     $fn = "$vfio_basedir/bind";
5366     if (! -d $testdir) {
5367         return undef if !file_write($fn, $name);
5368     }
5369
5370     return -d $testdir;
5371 }
5372
5373 sub pci_dev_group_bind_to_vfio {
5374     my ($pciid) = @_;
5375
5376     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5377
5378     if (!-d $vfio_basedir) {
5379         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5380     }
5381     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5382
5383     # get IOMMU group devices
5384     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5385       my @devs = grep /^0000:/, readdir($D);
5386     closedir($D);
5387
5388     foreach my $pciid (@devs) {
5389         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5390
5391         # pci bridges, switches or root ports are not supported
5392         # they have a pci_bus subdirectory so skip them
5393         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5394
5395         my $info = pci_device_info($1);
5396         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5397     }
5398
5399     return 1;
5400 }
5401
5402 # vzdump restore implementaion
5403
5404 sub tar_archive_read_firstfile {
5405     my $archive = shift;
5406
5407     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5408
5409     # try to detect archive type first
5410     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5411         die "unable to open file '$archive'\n";
5412     my $firstfile = <$fh>;
5413     kill 15, $pid;
5414     close $fh;
5415
5416     die "ERROR: archive contaions no data\n" if !$firstfile;
5417     chomp $firstfile;
5418
5419     return $firstfile;
5420 }
5421
5422 sub tar_restore_cleanup {
5423     my ($storecfg, $statfile) = @_;
5424
5425     print STDERR "starting cleanup\n";
5426
5427     if (my $fd = IO::File->new($statfile, "r")) {
5428         while (defined(my $line = <$fd>)) {
5429             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5430                 my $volid = $2;
5431                 eval {
5432                     if ($volid =~ m|^/|) {
5433                         unlink $volid || die 'unlink failed\n';
5434                     } else {
5435                         PVE::Storage::vdisk_free($storecfg, $volid);
5436                     }
5437                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5438                 };
5439                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5440             } else {
5441                 print STDERR "unable to parse line in statfile - $line";
5442             }
5443         }
5444         $fd->close();
5445     }
5446 }
5447
5448 sub restore_archive {
5449     my ($archive, $vmid, $user, $opts) = @_;
5450
5451     my $format = $opts->{format};
5452     my $comp;
5453
5454     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5455         $format = 'tar' if !$format;
5456         $comp = 'gzip';
5457     } elsif ($archive =~ m/\.tar$/) {
5458         $format = 'tar' if !$format;
5459     } elsif ($archive =~ m/.tar.lzo$/) {
5460         $format = 'tar' if !$format;
5461         $comp = 'lzop';
5462     } elsif ($archive =~ m/\.vma$/) {
5463         $format = 'vma' if !$format;
5464     } elsif ($archive =~ m/\.vma\.gz$/) {
5465         $format = 'vma' if !$format;
5466         $comp = 'gzip';
5467     } elsif ($archive =~ m/\.vma\.lzo$/) {
5468         $format = 'vma' if !$format;
5469         $comp = 'lzop';
5470     } else {
5471         $format = 'vma' if !$format; # default
5472     }
5473
5474     # try to detect archive format
5475     if ($format eq 'tar') {
5476         return restore_tar_archive($archive, $vmid, $user, $opts);
5477     } else {
5478         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5479     }
5480 }
5481
5482 sub restore_update_config_line {
5483     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5484
5485     return if $line =~ m/^\#qmdump\#/;
5486     return if $line =~ m/^\#vzdump\#/;
5487     return if $line =~ m/^lock:/;
5488     return if $line =~ m/^unused\d+:/;
5489     return if $line =~ m/^parent:/;
5490     return if $line =~ m/^template:/; # restored VM is never a template
5491
5492     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5493     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5494         # try to convert old 1.X settings
5495         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5496         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5497             my ($model, $macaddr) = split(/\=/, $devconfig);
5498             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5499             my $net = {
5500                 model => $model,
5501                 bridge => "vmbr$ind",
5502                 macaddr => $macaddr,
5503             };
5504             my $netstr = print_net($net);
5505
5506             print $outfd "net$cookie->{netcount}: $netstr\n";
5507             $cookie->{netcount}++;
5508         }
5509     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5510         my ($id, $netstr) = ($1, $2);
5511         my $net = parse_net($netstr);
5512         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5513         $netstr = print_net($net);
5514         print $outfd "$id: $netstr\n";
5515     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5516         my $virtdev = $1;
5517         my $value = $3;
5518         my $di = parse_drive($virtdev, $value);
5519         if (defined($di->{backup}) && !$di->{backup}) {
5520             print $outfd "#$line";
5521         } elsif ($map->{$virtdev}) {
5522             delete $di->{format}; # format can change on restore
5523             $di->{file} = $map->{$virtdev};
5524             $value = print_drive($vmid, $di);
5525             print $outfd "$virtdev: $value\n";
5526         } else {
5527             print $outfd $line;
5528         }
5529     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5530         my ($uuid, $uuid_str);
5531         UUID::generate($uuid);
5532         UUID::unparse($uuid, $uuid_str);
5533         my $smbios1 = parse_smbios1($2);
5534         $smbios1->{uuid} = $uuid_str;
5535         print $outfd $1.print_smbios1($smbios1)."\n";
5536     } else {
5537         print $outfd $line;
5538     }
5539 }
5540
5541 sub scan_volids {
5542     my ($cfg, $vmid) = @_;
5543
5544     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5545
5546     my $volid_hash = {};
5547     foreach my $storeid (keys %$info) {
5548         foreach my $item (@{$info->{$storeid}}) {
5549             next if !($item->{volid} && $item->{size});
5550             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5551             $volid_hash->{$item->{volid}} = $item;
5552         }
5553     }
5554
5555     return $volid_hash;
5556 }
5557
5558 sub is_volume_in_use {
5559     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5560
5561     my $path = PVE::Storage::path($storecfg, $volid);
5562
5563     my $scan_config = sub {
5564         my ($cref, $snapname) = @_;
5565
5566         foreach my $key (keys %$cref) {
5567             my $value = $cref->{$key};
5568             if (is_valid_drivename($key)) {
5569                 next if $skip_drive && $key eq $skip_drive;
5570                 my $drive = parse_drive($key, $value);
5571                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5572                 return 1 if $volid eq $drive->{file};
5573                 if ($drive->{file} =~ m!^/!) {
5574                     return 1 if $drive->{file} eq $path;
5575                 } else {
5576                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5577                     next if !$storeid;
5578                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5579                     next if !$scfg;
5580                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5581                 }
5582             }
5583         }
5584
5585         return 0;
5586     };
5587
5588     return 1 if &$scan_config($conf);
5589
5590     undef $skip_drive;
5591
5592     foreach my $snapname (keys %{$conf->{snapshots}}) {
5593         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5594     }
5595
5596     return 0;
5597 }
5598
5599 sub update_disksize {
5600     my ($vmid, $conf, $volid_hash) = @_;
5601
5602     my $changes;
5603
5604     # used and unused disks
5605     my $referenced = {};
5606
5607     # Note: it is allowed to define multiple storages with same path (alias), so
5608     # we need to check both 'volid' and real 'path' (two different volid can point
5609     # to the same path).
5610
5611     my $referencedpath = {};
5612
5613     # update size info
5614     foreach my $opt (keys %$conf) {
5615         if (is_valid_drivename($opt)) {
5616             my $drive = parse_drive($opt, $conf->{$opt});
5617             my $volid = $drive->{file};
5618             next if !$volid;
5619
5620             $referenced->{$volid} = 1;
5621             if ($volid_hash->{$volid} &&
5622                 (my $path = $volid_hash->{$volid}->{path})) {
5623                 $referencedpath->{$path} = 1;
5624             }
5625
5626             next if drive_is_cdrom($drive);
5627             next if !$volid_hash->{$volid};
5628
5629             $drive->{size} = $volid_hash->{$volid}->{size};
5630             my $new = print_drive($vmid, $drive);
5631             if ($new ne $conf->{$opt}) {
5632                 $changes = 1;
5633                 $conf->{$opt} = $new;
5634             }
5635         }
5636     }
5637
5638     # remove 'unusedX' entry if volume is used
5639     foreach my $opt (keys %$conf) {
5640         next if $opt !~ m/^unused\d+$/;
5641         my $volid = $conf->{$opt};
5642         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5643         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5644             $changes = 1;
5645             delete $conf->{$opt};
5646         }
5647
5648         $referenced->{$volid} = 1;
5649         $referencedpath->{$path} = 1 if $path;
5650     }
5651
5652     foreach my $volid (sort keys %$volid_hash) {
5653         next if $volid =~ m/vm-$vmid-state-/;
5654         next if $referenced->{$volid};
5655         my $path = $volid_hash->{$volid}->{path};
5656         next if !$path; # just to be sure
5657         next if $referencedpath->{$path};
5658         $changes = 1;
5659         PVE::QemuConfig->add_unused_volume($conf, $volid);
5660         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5661     }
5662
5663     return $changes;
5664 }
5665
5666 sub rescan {
5667     my ($vmid, $nolock) = @_;
5668
5669     my $cfg = PVE::Storage::config();
5670
5671     my $volid_hash = scan_volids($cfg, $vmid);
5672
5673     my $updatefn =  sub {
5674         my ($vmid) = @_;
5675
5676         my $conf = PVE::QemuConfig->load_config($vmid);
5677
5678         PVE::QemuConfig->check_lock($conf);
5679
5680         my $vm_volids = {};
5681         foreach my $volid (keys %$volid_hash) {
5682             my $info = $volid_hash->{$volid};
5683             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5684         }
5685
5686         my $changes = update_disksize($vmid, $conf, $vm_volids);
5687
5688         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5689     };
5690
5691     if (defined($vmid)) {
5692         if ($nolock) {
5693             &$updatefn($vmid);
5694         } else {
5695             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5696         }
5697     } else {
5698         my $vmlist = config_list();
5699         foreach my $vmid (keys %$vmlist) {
5700             if ($nolock) {
5701                 &$updatefn($vmid);
5702             } else {
5703                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5704             }
5705         }
5706     }
5707 }
5708
5709 sub restore_vma_archive {
5710     my ($archive, $vmid, $user, $opts, $comp) = @_;
5711
5712     my $input = $archive eq '-' ? "<&STDIN" : undef;
5713     my $readfrom = $archive;
5714
5715     my $uncomp = '';
5716     if ($comp) {
5717         $readfrom = '-';
5718         my $qarchive = PVE::Tools::shellquote($archive);
5719         if ($comp eq 'gzip') {
5720             $uncomp = "zcat $qarchive|";
5721         } elsif ($comp eq 'lzop') {
5722             $uncomp = "lzop -d -c $qarchive|";
5723         } else {
5724             die "unknown compression method '$comp'\n";
5725         }
5726
5727     }
5728
5729     my $tmpdir = "/var/tmp/vzdumptmp$$";
5730     rmtree $tmpdir;
5731
5732     # disable interrupts (always do cleanups)
5733     local $SIG{INT} =
5734         local $SIG{TERM} =
5735         local $SIG{QUIT} =
5736         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5737
5738     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5739     POSIX::mkfifo($mapfifo, 0600);
5740     my $fifofh;
5741
5742     my $openfifo = sub {
5743         open($fifofh, '>', $mapfifo) || die $!;
5744     };
5745
5746     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5747
5748     my $oldtimeout;
5749     my $timeout = 5;
5750
5751     my $devinfo = {};
5752
5753     my $rpcenv = PVE::RPCEnvironment::get();
5754
5755     my $conffile = PVE::QemuConfig->config_file($vmid);
5756     my $tmpfn = "$conffile.$$.tmp";
5757
5758     # Note: $oldconf is undef if VM does not exists
5759     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5760     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5761
5762     my $print_devmap = sub {
5763         my $virtdev_hash = {};
5764
5765         my $cfgfn = "$tmpdir/qemu-server.conf";
5766
5767         # we can read the config - that is already extracted
5768         my $fh = IO::File->new($cfgfn, "r") ||
5769             "unable to read qemu-server.conf - $!\n";
5770
5771         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5772         if (-f $fwcfgfn) {
5773             my $pve_firewall_dir = '/etc/pve/firewall';
5774             mkdir $pve_firewall_dir; # make sure the dir exists
5775             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5776         }
5777
5778         while (defined(my $line = <$fh>)) {
5779             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5780                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5781                 die "archive does not contain data for drive '$virtdev'\n"
5782                     if !$devinfo->{$devname};
5783                 if (defined($opts->{storage})) {
5784                     $storeid = $opts->{storage} || 'local';
5785                 } elsif (!$storeid) {
5786                     $storeid = 'local';
5787                 }
5788                 $format = 'raw' if !$format;
5789                 $devinfo->{$devname}->{devname} = $devname;
5790                 $devinfo->{$devname}->{virtdev} = $virtdev;
5791                 $devinfo->{$devname}->{format} = $format;
5792                 $devinfo->{$devname}->{storeid} = $storeid;
5793
5794                 # check permission on storage
5795                 my $pool = $opts->{pool}; # todo: do we need that?
5796                 if ($user ne 'root@pam') {
5797                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5798                 }
5799
5800                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5801             }
5802         }
5803
5804         foreach my $devname (keys %$devinfo) {
5805             die "found no device mapping information for device '$devname'\n"
5806                 if !$devinfo->{$devname}->{virtdev};
5807         }
5808
5809         my $cfg = PVE::Storage::config();
5810
5811         # create empty/temp config
5812         if ($oldconf) {
5813             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5814             foreach_drive($oldconf, sub {
5815                 my ($ds, $drive) = @_;
5816
5817                 return if drive_is_cdrom($drive);
5818
5819                 my $volid = $drive->{file};
5820
5821                 return if !$volid || $volid =~ m|^/|;
5822
5823                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5824                 return if !$path || !$owner || ($owner != $vmid);
5825
5826                 # Note: only delete disk we want to restore
5827                 # other volumes will become unused
5828                 if ($virtdev_hash->{$ds}) {
5829                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5830                     if (my $err = $@) {
5831                         warn $err;
5832                     }
5833                 }
5834             });
5835
5836             # delete vmstate files
5837             # since after the restore we have no snapshots anymore
5838             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5839                 my $snap = $oldconf->{snapshots}->{$snapname};
5840                 if ($snap->{vmstate}) {
5841                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5842                     if (my $err = $@) {
5843                         warn $err;
5844                     }
5845                 }
5846             }
5847         }
5848
5849         my $map = {};
5850         foreach my $virtdev (sort keys %$virtdev_hash) {
5851             my $d = $virtdev_hash->{$virtdev};
5852             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5853             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5854
5855             # test if requested format is supported
5856             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5857             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5858             $d->{format} = $defFormat if !$supported;
5859
5860             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5861                                                   $d->{format}, undef, $alloc_size);
5862             print STDERR "new volume ID is '$volid'\n";
5863             $d->{volid} = $volid;
5864             my $path = PVE::Storage::path($cfg, $volid);
5865
5866             PVE::Storage::activate_volumes($cfg,[$volid]);
5867
5868             my $write_zeros = 1;
5869             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5870                 $write_zeros = 0;
5871             }
5872
5873             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5874
5875             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5876             $map->{$virtdev} = $volid;
5877         }
5878
5879         $fh->seek(0, 0) || die "seek failed - $!\n";
5880
5881         my $outfd = new IO::File ($tmpfn, "w") ||
5882             die "unable to write config for VM $vmid\n";
5883
5884         my $cookie = { netcount => 0 };
5885         while (defined(my $line = <$fh>)) {
5886             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5887         }
5888
5889         $fh->close();
5890         $outfd->close();
5891     };
5892
5893     eval {
5894         # enable interrupts
5895         local $SIG{INT} =
5896             local $SIG{TERM} =
5897             local $SIG{QUIT} =
5898             local $SIG{HUP} =
5899             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5900         local $SIG{ALRM} = sub { die "got timeout\n"; };
5901
5902         $oldtimeout = alarm($timeout);
5903
5904         my $parser = sub {
5905             my $line = shift;
5906
5907             print "$line\n";
5908
5909             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5910                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5911                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5912             } elsif ($line =~ m/^CTIME: /) {
5913                 # we correctly received the vma config, so we can disable
5914                 # the timeout now for disk allocation (set to 10 minutes, so
5915                 # that we always timeout if something goes wrong)
5916                 alarm(600);
5917                 &$print_devmap();
5918                 print $fifofh "done\n";
5919                 my $tmp = $oldtimeout || 0;
5920                 $oldtimeout = undef;
5921                 alarm($tmp);
5922                 close($fifofh);
5923             }
5924         };
5925
5926         print "restore vma archive: $cmd\n";
5927         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5928     };
5929     my $err = $@;
5930
5931     alarm($oldtimeout) if $oldtimeout;
5932
5933     my $vollist = [];
5934     foreach my $devname (keys %$devinfo) {
5935         my $volid = $devinfo->{$devname}->{volid};
5936         push @$vollist, $volid if $volid;
5937     }
5938
5939     my $cfg = PVE::Storage::config();
5940     PVE::Storage::deactivate_volumes($cfg, $vollist);
5941
5942     unlink $mapfifo;
5943
5944     if ($err) {
5945         rmtree $tmpdir;
5946         unlink $tmpfn;
5947
5948         foreach my $devname (keys %$devinfo) {
5949             my $volid = $devinfo->{$devname}->{volid};
5950             next if !$volid;
5951             eval {
5952                 if ($volid =~ m|^/|) {
5953                     unlink $volid || die 'unlink failed\n';
5954                 } else {
5955                     PVE::Storage::vdisk_free($cfg, $volid);
5956                 }
5957                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5958             };
5959             print STDERR "unable to cleanup '$volid' - $@" if $@;
5960         }
5961         die $err;
5962     }
5963
5964     rmtree $tmpdir;
5965
5966     rename($tmpfn, $conffile) ||
5967         die "unable to commit configuration file '$conffile'\n";
5968
5969     PVE::Cluster::cfs_update(); # make sure we read new file
5970
5971     eval { rescan($vmid, 1); };
5972     warn $@ if $@;
5973 }
5974
5975 sub restore_tar_archive {
5976     my ($archive, $vmid, $user, $opts) = @_;
5977
5978     if ($archive ne '-') {
5979         my $firstfile = tar_archive_read_firstfile($archive);
5980         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5981             if $firstfile ne 'qemu-server.conf';
5982     }
5983
5984     my $storecfg = PVE::Storage::config();
5985
5986     # destroy existing data - keep empty config
5987     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5988     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5989
5990     my $tocmd = "/usr/lib/qemu-server/qmextract";
5991
5992     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5993     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5994     $tocmd .= ' --prealloc' if $opts->{prealloc};
5995     $tocmd .= ' --info' if $opts->{info};
5996
5997     # tar option "xf" does not autodetect compression when read from STDIN,
5998     # so we pipe to zcat
5999     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6000         PVE::Tools::shellquote("--to-command=$tocmd");
6001
6002     my $tmpdir = "/var/tmp/vzdumptmp$$";
6003     mkpath $tmpdir;
6004
6005     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6006     local $ENV{VZDUMP_VMID} = $vmid;
6007     local $ENV{VZDUMP_USER} = $user;
6008
6009     my $conffile = PVE::QemuConfig->config_file($vmid);
6010     my $tmpfn = "$conffile.$$.tmp";
6011
6012     # disable interrupts (always do cleanups)
6013     local $SIG{INT} =
6014         local $SIG{TERM} =
6015         local $SIG{QUIT} =
6016         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6017
6018     eval {
6019         # enable interrupts
6020         local $SIG{INT} =
6021             local $SIG{TERM} =
6022             local $SIG{QUIT} =
6023             local $SIG{HUP} =
6024             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6025
6026         if ($archive eq '-') {
6027             print "extracting archive from STDIN\n";
6028             run_command($cmd, input => "<&STDIN");
6029         } else {
6030             print "extracting archive '$archive'\n";
6031             run_command($cmd);
6032         }
6033
6034         return if $opts->{info};
6035
6036         # read new mapping
6037         my $map = {};
6038         my $statfile = "$tmpdir/qmrestore.stat";
6039         if (my $fd = IO::File->new($statfile, "r")) {
6040             while (defined (my $line = <$fd>)) {
6041                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6042                     $map->{$1} = $2 if $1;
6043                 } else {
6044                     print STDERR "unable to parse line in statfile - $line\n";
6045                 }
6046             }
6047             $fd->close();
6048         }
6049
6050         my $confsrc = "$tmpdir/qemu-server.conf";
6051
6052         my $srcfd = new IO::File($confsrc, "r") ||
6053             die "unable to open file '$confsrc'\n";
6054
6055         my $outfd = new IO::File ($tmpfn, "w") ||
6056             die "unable to write config for VM $vmid\n";
6057
6058         my $cookie = { netcount => 0 };
6059         while (defined (my $line = <$srcfd>)) {
6060             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6061         }
6062
6063         $srcfd->close();
6064         $outfd->close();
6065     };
6066     my $err = $@;
6067
6068     if ($err) {
6069
6070         unlink $tmpfn;
6071
6072         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6073
6074         die $err;
6075     }
6076
6077     rmtree $tmpdir;
6078
6079     rename $tmpfn, $conffile ||
6080         die "unable to commit configuration file '$conffile'\n";
6081
6082     PVE::Cluster::cfs_update(); # make sure we read new file
6083
6084     eval { rescan($vmid, 1); };
6085     warn $@ if $@;
6086 };
6087
6088 sub foreach_storage_used_by_vm {
6089     my ($conf, $func) = @_;
6090
6091     my $sidhash = {};
6092
6093     foreach_drive($conf, sub {
6094         my ($ds, $drive) = @_;
6095         return if drive_is_cdrom($drive);
6096
6097         my $volid = $drive->{file};
6098
6099         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6100         $sidhash->{$sid} = $sid if $sid;
6101     });
6102
6103     foreach my $sid (sort keys %$sidhash) {
6104         &$func($sid);
6105     }
6106 }
6107
6108 sub do_snapshots_with_qemu {
6109     my ($storecfg, $volid) = @_;
6110
6111     my $storage_name = PVE::Storage::parse_volume_id($volid);
6112
6113     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6114         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6115         return 1;
6116     }
6117
6118     if ($volid =~ m/\.(qcow2|qed)$/){
6119         return 1;
6120     }
6121
6122     return undef;
6123 }
6124
6125 sub qga_check_running {
6126     my ($vmid) = @_;
6127
6128     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6129     if ($@) {
6130         warn "Qemu Guest Agent is not running - $@";
6131         return 0;
6132     }
6133     return 1;
6134 }
6135
6136 sub template_create {
6137     my ($vmid, $conf, $disk) = @_;
6138
6139     my $storecfg = PVE::Storage::config();
6140
6141     foreach_drive($conf, sub {
6142         my ($ds, $drive) = @_;
6143
6144         return if drive_is_cdrom($drive);
6145         return if $disk && $ds ne $disk;
6146
6147         my $volid = $drive->{file};
6148         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6149
6150         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6151         $drive->{file} = $voliddst;
6152         $conf->{$ds} = print_drive($vmid, $drive);
6153         PVE::QemuConfig->write_config($vmid, $conf);
6154     });
6155 }
6156
6157 sub qemu_img_convert {
6158     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6159
6160     my $storecfg = PVE::Storage::config();
6161     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6162     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6163
6164     if ($src_storeid && $dst_storeid) {
6165
6166         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6167
6168         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6169         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6170
6171         my $src_format = qemu_img_format($src_scfg, $src_volname);
6172         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6173
6174         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6175         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6176
6177         my $cmd = [];
6178         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6179         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6180         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6181         if ($is_zero_initialized) {
6182             push @$cmd, "zeroinit:$dst_path";
6183         } else {
6184             push @$cmd, $dst_path;
6185         }
6186
6187         my $parser = sub {
6188             my $line = shift;
6189             if($line =~ m/\((\S+)\/100\%\)/){
6190                 my $percent = $1;
6191                 my $transferred = int($size * $percent / 100);
6192                 my $remaining = $size - $transferred;
6193
6194                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6195             }
6196
6197         };
6198
6199         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6200         my $err = $@;
6201         die "copy failed: $err" if $err;
6202     }
6203 }
6204
6205 sub qemu_img_format {
6206     my ($scfg, $volname) = @_;
6207
6208     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6209         return $1;
6210     } else {
6211         return "raw";
6212     }
6213 }
6214
6215 sub qemu_drive_mirror {
6216     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6217
6218     $jobs = {} if !$jobs;
6219
6220     my $qemu_target;
6221     my $format;
6222     $jobs->{"drive-$drive"} = {};
6223
6224     if ($dst_volid =~ /^nbd:/) {
6225         $qemu_target = $dst_volid;
6226         $format = "nbd";
6227     } else {
6228         my $storecfg = PVE::Storage::config();
6229         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6230
6231         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6232
6233         $format = qemu_img_format($dst_scfg, $dst_volname);
6234
6235         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6236
6237         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6238     }
6239
6240     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6241     $opts->{format} = $format if $format;
6242
6243     print "drive mirror is starting for drive-$drive\n";
6244
6245     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6246
6247     if (my $err = $@) {
6248         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6249         die "mirroring error: $err";
6250     }
6251
6252     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6253 }
6254
6255 sub qemu_drive_mirror_monitor {
6256     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6257
6258     eval {
6259         my $err_complete = 0;
6260
6261         while (1) {
6262             die "storage migration timed out\n" if $err_complete > 300;
6263
6264             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6265
6266             my $running_mirror_jobs = {};
6267             foreach my $stat (@$stats) {
6268                 next if $stat->{type} ne 'mirror';
6269                 $running_mirror_jobs->{$stat->{device}} = $stat;
6270             }
6271
6272             my $readycounter = 0;
6273
6274             foreach my $job (keys %$jobs) {
6275
6276                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6277                     print "$job : finished\n";
6278                     delete $jobs->{$job};
6279                     next;
6280                 }
6281
6282                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6283
6284                 my $busy = $running_mirror_jobs->{$job}->{busy};
6285                 my $ready = $running_mirror_jobs->{$job}->{ready};
6286                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6287                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6288                     my $remaining = $total - $transferred;
6289                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6290
6291                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6292                 }
6293
6294                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6295             }
6296
6297             last if scalar(keys %$jobs) == 0;
6298
6299             if ($readycounter == scalar(keys %$jobs)) {
6300                 print "all mirroring jobs are ready \n";
6301                 last if $skipcomplete; #do the complete later
6302
6303                 if ($vmiddst && $vmiddst != $vmid) {
6304                     my $agent_running = $qga && qga_check_running($vmid);
6305                     if ($agent_running) {
6306                         print "freeze filesystem\n";
6307                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6308                     } else {
6309                         print "suspend vm\n";
6310                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6311                     }
6312
6313                     # if we clone a disk for a new target vm, we don't switch the disk
6314                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6315
6316                     if ($agent_running) {
6317                         print "unfreeze filesystem\n";
6318                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6319                     } else {
6320                         print "resume vm\n";
6321                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6322                     }
6323
6324                     last;
6325                 } else {
6326
6327                     foreach my $job (keys %$jobs) {
6328                         # try to switch the disk if source and destination are on the same guest
6329                         print "$job: Completing block job...\n";
6330
6331                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6332                         if ($@ =~ m/cannot be completed/) {
6333                             print "$job: Block job cannot be completed, try again.\n";
6334                             $err_complete++;
6335                         }else {
6336                             print "$job: Completed successfully.\n";
6337                             $jobs->{$job}->{complete} = 1;
6338                         }
6339                     }
6340                 }
6341             }
6342             sleep 1;
6343         }
6344     };
6345     my $err = $@;
6346
6347     if ($err) {
6348         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6349         die "mirroring error: $err";
6350     }
6351
6352 }
6353
6354 sub qemu_blockjobs_cancel {
6355     my ($vmid, $jobs) = @_;
6356
6357     foreach my $job (keys %$jobs) {
6358         print "$job: Cancelling block job\n";
6359         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6360         $jobs->{$job}->{cancel} = 1;
6361     }
6362
6363     while (1) {
6364         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6365
6366         my $running_jobs = {};
6367         foreach my $stat (@$stats) {
6368             $running_jobs->{$stat->{device}} = $stat;
6369         }
6370
6371         foreach my $job (keys %$jobs) {
6372
6373             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6374                 print "$job: Done.\n";
6375                 delete $jobs->{$job};
6376             }
6377         }
6378
6379         last if scalar(keys %$jobs) == 0;
6380
6381         sleep 1;
6382     }
6383 }
6384
6385 sub clone_disk {
6386     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6387         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6388
6389     my $newvolid;
6390
6391     if (!$full) {
6392         print "create linked clone of drive $drivename ($drive->{file})\n";
6393         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6394         push @$newvollist, $newvolid;
6395     } else {
6396
6397         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6398         $storeid = $storage if $storage;
6399
6400         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6401         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6402
6403         print "create full clone of drive $drivename ($drive->{file})\n";
6404         my $name = undef;
6405         if (drive_is_cloudinit($drive)) {
6406             $name = "vm-$newvmid-cloudinit";
6407             # cloudinit only supports raw and qcow2 atm:
6408             if ($dst_format eq 'qcow2') {
6409                 $name .= '.qcow2';
6410             } elsif ($dst_format ne 'raw') {
6411                 die "clone: unhandled format for cloudinit image\n";
6412             }
6413         }
6414         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6415         push @$newvollist, $newvolid;
6416
6417         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6418
6419         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6420         if (!$running || $snapname) {
6421             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6422         } else {
6423
6424             my $kvmver = get_running_qemu_version ($vmid);
6425             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6426                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6427                     if $drive->{iothread};
6428             }
6429
6430             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6431         }
6432     }
6433
6434     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6435
6436     my $disk = $drive;
6437     $disk->{format} = undef;
6438     $disk->{file} = $newvolid;
6439     $disk->{size} = $size;
6440
6441     return $disk;
6442 }
6443
6444 # this only works if VM is running
6445 sub get_current_qemu_machine {
6446     my ($vmid) = @_;
6447
6448     my $cmd = { execute => 'query-machines', arguments => {} };
6449     my $res = vm_qmp_command($vmid, $cmd);
6450
6451     my ($current, $default);
6452     foreach my $e (@$res) {
6453         $default = $e->{name} if $e->{'is-default'};
6454         $current = $e->{name} if $e->{'is-current'};
6455     }
6456
6457     # fallback to the default machine if current is not supported by qemu
6458     return $current || $default || 'pc';
6459 }
6460
6461 sub get_running_qemu_version {
6462     my ($vmid) = @_;
6463     my $cmd = { execute => 'query-version', arguments => {} };
6464     my $res = vm_qmp_command($vmid, $cmd);
6465     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6466 }
6467
6468 sub qemu_machine_feature_enabled {
6469     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6470
6471     my $current_major;
6472     my $current_minor;
6473
6474     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6475
6476         $current_major = $3;
6477         $current_minor = $4;
6478
6479     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6480
6481         $current_major = $1;
6482         $current_minor = $2;
6483     }
6484
6485     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6486
6487
6488 }
6489
6490 sub qemu_machine_pxe {
6491     my ($vmid, $conf, $machine) = @_;
6492
6493     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6494
6495     foreach my $opt (keys %$conf) {
6496         next if $opt !~ m/^net(\d+)$/;
6497         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6498         next if !$net;
6499         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6500         return $machine.".pxe" if $romfile =~ m/pxe/;
6501         last;
6502     }
6503
6504     return $machine;
6505 }
6506
6507 sub qemu_use_old_bios_files {
6508     my ($machine_type) = @_;
6509
6510     return if !$machine_type;
6511
6512     my $use_old_bios_files = undef;
6513
6514     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6515         $machine_type = $1;
6516         $use_old_bios_files = 1;
6517     } else {
6518         my $kvmver = kvm_user_version();
6519         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6520         # load new efi bios files on migration. So this hack is required to allow
6521         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6522         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6523         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6524     }
6525
6526     return ($use_old_bios_files, $machine_type);
6527 }
6528
6529 sub create_efidisk {
6530     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6531
6532     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6533
6534     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6535     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6536     PVE::Storage::activate_volumes($storecfg, [$volid]);
6537
6538     my $path = PVE::Storage::path($storecfg, $volid);
6539     eval {
6540         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6541     };
6542     die "Copying EFI vars image failed: $@" if $@;
6543
6544     return ($volid, $vars_size);
6545 }
6546
6547 sub lspci {
6548
6549     my $devices = {};
6550
6551     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6552             my (undef, $id, $function) = @_;
6553             my $res = { id => $id, function => $function};
6554             push @{$devices->{$id}}, $res;
6555     });
6556
6557     # Entries should be sorted by functions.
6558     foreach my $id (keys %$devices) {
6559         my $dev = $devices->{$id};
6560         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6561     }
6562
6563     return $devices;
6564 }
6565
6566 sub vm_iothreads_list {
6567     my ($vmid) = @_;
6568
6569     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6570
6571     my $iothreads = {};
6572     foreach my $iothread (@$res) {
6573         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6574     }
6575
6576     return $iothreads;
6577 }
6578
6579 sub scsihw_infos {
6580     my ($conf, $drive) = @_;
6581
6582     my $maxdev = 0;
6583
6584     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6585         $maxdev = 7;
6586     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6587         $maxdev = 1;
6588     } else {
6589         $maxdev = 256;
6590     }
6591
6592     my $controller = int($drive->{index} / $maxdev);
6593     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6594
6595     return ($maxdev, $controller, $controller_prefix);
6596 }
6597
6598 sub add_hyperv_enlightenments {
6599     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6600
6601     return if $winversion < 6;
6602     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6603
6604     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6605
6606     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6607         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6608         push @$cpuFlags , 'hv_vapic';
6609         push @$cpuFlags , 'hv_time';
6610     } else {
6611         push @$cpuFlags , 'hv_spinlocks=0xffff';
6612     }
6613
6614     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6615         push @$cpuFlags , 'hv_reset';
6616         push @$cpuFlags , 'hv_vpindex';
6617         push @$cpuFlags , 'hv_runtime';
6618     }
6619
6620     if ($winversion >= 7) {
6621         push @$cpuFlags , 'hv_relaxed';
6622     }
6623 }
6624
6625 sub windows_version {
6626     my ($ostype) = @_;
6627
6628     return 0 if !$ostype;
6629
6630     my $winversion = 0;
6631
6632     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6633         $winversion = 5;
6634     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6635         $winversion = 6;
6636     } elsif ($ostype =~ m/^win(\d+)$/) {
6637         $winversion = $1;
6638     }
6639
6640     return $winversion;
6641 }
6642
6643 sub resolve_dst_disk_format {
6644         my ($storecfg, $storeid, $src_volname, $format) = @_;
6645         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6646
6647         if (!$format) {
6648             # if no target format is specified, use the source disk format as hint
6649             if ($src_volname) {
6650                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6651                 $format = qemu_img_format($scfg, $src_volname);
6652             } else {
6653                 return $defFormat;
6654             }
6655         }
6656
6657         # test if requested format is supported - else use default
6658         my $supported = grep { $_ eq $format } @$validFormats;
6659         $format = $defFormat if !$supported;
6660         return $format;
6661 }
6662
6663 sub resolve_first_disk {
6664     my $conf = shift;
6665     my @disks = PVE::QemuServer::valid_drive_names();
6666     my $firstdisk;
6667     foreach my $ds (reverse @disks) {
6668         next if !$conf->{$ds};
6669         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6670         next if PVE::QemuServer::drive_is_cdrom($disk);
6671         $firstdisk = $ds;
6672     }
6673     return $firstdisk;
6674 }
6675
6676 sub generate_smbios1_uuid {
6677     my ($uuid, $uuid_str);
6678     UUID::generate($uuid);
6679     UUID::unparse($uuid, $uuid_str);
6680     return "uuid=$uuid_str";
6681 }
6682
6683 # bash completion helper
6684
6685 sub complete_backup_archives {
6686     my ($cmdname, $pname, $cvalue) = @_;
6687
6688     my $cfg = PVE::Storage::config();
6689
6690     my $storeid;
6691
6692     if ($cvalue =~ m/^([^:]+):/) {
6693         $storeid = $1;
6694     }
6695
6696     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6697
6698     my $res = [];
6699     foreach my $id (keys %$data) {
6700         foreach my $item (@{$data->{$id}}) {
6701             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6702             push @$res, $item->{volid} if defined($item->{volid});
6703         }
6704     }
6705
6706     return $res;
6707 }
6708
6709 my $complete_vmid_full = sub {
6710     my ($running) = @_;
6711
6712     my $idlist = vmstatus();
6713
6714     my $res = [];
6715
6716     foreach my $id (keys %$idlist) {
6717         my $d = $idlist->{$id};
6718         if (defined($running)) {
6719             next if $d->{template};
6720             next if $running && $d->{status} ne 'running';
6721             next if !$running && $d->{status} eq 'running';
6722         }
6723         push @$res, $id;
6724
6725     }
6726     return $res;
6727 };
6728
6729 sub complete_vmid {
6730     return &$complete_vmid_full();
6731 }
6732
6733 sub complete_vmid_stopped {
6734     return &$complete_vmid_full(0);
6735 }
6736
6737 sub complete_vmid_running {
6738     return &$complete_vmid_full(1);
6739 }
6740
6741 sub complete_storage {
6742
6743     my $cfg = PVE::Storage::config();
6744     my $ids = $cfg->{ids};
6745
6746     my $res = [];
6747     foreach my $sid (keys %$ids) {
6748         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6749         next if !$ids->{$sid}->{content}->{images};
6750         push @$res, $sid;
6751     }
6752
6753     return $res;
6754 }
6755
6756 1;