PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     citype => {
 544         optional => 1,
 545         type => 'string',
 546         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 547         enum => ['configdrive2', 'nocloud'],
 548     },
 549     ciuser => {
 550         optional => 1,
 551         type => 'string',
 552         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 553     },
 554     cipassword => {
 555         optional => 1,
 556         type => 'string',
 557         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 558     },
 559     searchdomain => {
 560         optional => 1,
 561         type => 'string',
 562         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 563     },
 564     nameserver => {
 565         optional => 1,
 566         type => 'string', format => 'address-list',
 567         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 568     },
 569     sshkeys => {
 570         optional => 1,
 571         type => 'string',
 572         format => 'urlencoded',
 573         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 574     },
 575 };
 576
 577 # what about other qemu settings ?
 578 #cpu => 'string',
 579 #machine => 'string',
 580 #fda => 'file',
 581 #fdb => 'file',
 582 #mtdblock => 'file',
 583 #sd => 'file',
 584 #pflash => 'file',
 585 #snapshot => 'bool',
 586 #bootp => 'file',
 587 ##tftp => 'dir',
 588 ##smb => 'dir',
 589 #kernel => 'file',
 590 #append => 'string',
 591 #initrd => 'file',
 592 ##soundhw => 'string',
 593
 594 while (my ($k, $v) = each %$confdesc) {
 595     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 596 }
 597
 598 my $MAX_IDE_DISKS = 4;
 599 my $MAX_SCSI_DISKS = 14;
 600 my $MAX_VIRTIO_DISKS = 16;
 601 my $MAX_SATA_DISKS = 6;
 602 my $MAX_USB_DEVICES = 5;
 603 my $MAX_NETS = 32;
 604 my $MAX_UNUSED_DISKS = 8;
 605 my $MAX_HOSTPCI_DEVICES = 4;
 606 my $MAX_SERIAL_PORTS = 4;
 607 my $MAX_PARALLEL_PORTS = 3;
 608 my $MAX_NUMA = 8;
 609
 610 my $numa_fmt = {
 611     cpus => {
 612         type => "string",
 613         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 614         description => "CPUs accessing this NUMA node.",
 615         format_description => "id[-id];...",
 616     },
 617     memory => {
 618         type => "number",
 619         description => "Amount of memory this NUMA node provides.",
 620         optional => 1,
 621     },
 622     hostnodes => {
 623         type => "string",
 624         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 625         description => "Host NUMA nodes to use.",
 626         format_description => "id[-id];...",
 627         optional => 1,
 628     },
 629     policy => {
 630         type => 'string',
 631         enum => [qw(preferred bind interleave)],
 632         description => "NUMA allocation policy.",
 633         optional => 1,
 634     },
 635 };
 636 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 637 my $numadesc = {
 638     optional => 1,
 639     type => 'string', format => $numa_fmt,
 640     description => "NUMA topology.",
 641 };
 642 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 643
 644 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 645     $confdesc->{"numa$i"} = $numadesc;
 646 }
 647
 648 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 649                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 650                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 651 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 652
 653 my $net_fmt_bridge_descr = <<__EOD__;
 654 Bridge to attach the network device to. The Proxmox VE standard bridge
 655 is called 'vmbr0'.
 656
 657 If you do not specify a bridge, we create a kvm user (NATed) network
 658 device, which provides DHCP and DNS services. The following addresses
 659 are used:
 660
 661  10.0.2.2   Gateway
 662  10.0.2.3   DNS Server
 663  10.0.2.4   SMB Server
 664
 665 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 666 __EOD__
 667
 668 my $net_fmt = {
 669     macaddr => {
 670         type => 'string',
 671         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 672         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 673         format_description => "XX:XX:XX:XX:XX:XX",
 674         optional => 1,
 675     },
 676     model => {
 677         type => 'string',
 678         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 679         enum => $nic_model_list,
 680         default_key => 1,
 681     },
 682     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 683     bridge => {
 684         type => 'string',
 685         description => $net_fmt_bridge_descr,
 686         format_description => 'bridge',
 687         optional => 1,
 688     },
 689     queues => {
 690         type => 'integer',
 691         minimum => 0, maximum => 16,
 692         description => 'Number of packet queues to be used on the device.',
 693         optional => 1,
 694     },
 695     rate => {
 696         type => 'number',
 697         minimum => 0,
 698         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 699         optional => 1,
 700     },
 701     tag => {
 702         type => 'integer',
 703         minimum => 1, maximum => 4094,
 704         description => 'VLAN tag to apply to packets on this interface.',
 705         optional => 1,
 706     },
 707     trunks => {
 708         type => 'string',
 709         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 710         description => 'VLAN trunks to pass through this interface.',
 711         format_description => 'vlanid[;vlanid...]',
 712         optional => 1,
 713     },
 714     firewall => {
 715         type => 'boolean',
 716         description => 'Whether this interface should be protected by the firewall.',
 717         optional => 1,
 718     },
 719     link_down => {
 720         type => 'boolean',
 721         description => 'Whether this interface should be disconnected (like pulling the plug).',
 722         optional => 1,
 723     },
 724 };
 725
 726 my $netdesc = {
 727     optional => 1,
 728     type => 'string', format => $net_fmt,
 729     description => "Specify network devices.",
 730 };
 731
 732 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 733
 734 my $ipconfig_fmt = {
 735     ip => {
 736         type => 'string',
 737         format => 'pve-ipv4-config',
 738         format_description => 'IPv4Format/CIDR',
 739         description => 'IPv4 address in CIDR format.',
 740         optional => 1,
 741         default => 'dhcp',
 742     },
 743     gw => {
 744         type => 'string',
 745         format => 'ipv4',
 746         format_description => 'GatewayIPv4',
 747         description => 'Default gateway for IPv4 traffic.',
 748         optional => 1,
 749         requires => 'ip',
 750     },
 751     ip6 => {
 752         type => 'string',
 753         format => 'pve-ipv6-config',
 754         format_description => 'IPv6Format/CIDR',
 755         description => 'IPv6 address in CIDR format.',
 756         optional => 1,
 757         default => 'dhcp',
 758     },
 759     gw6 => {
 760         type => 'string',
 761         format => 'ipv6',
 762         format_description => 'GatewayIPv6',
 763         description => 'Default gateway for IPv6 traffic.',
 764         optional => 1,
 765         requires => 'ip6',
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 769 my $ipconfigdesc = {
 770     optional => 1,
 771     type => 'string', format => 'pve-qm-ipconfig',
 772     description => <<'EODESCR',
 773 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 774
 775 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 776
 777 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 778 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 779
 780 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 781 EODESCR
 782 };
 783 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 784
 785 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 786     $confdesc->{"net$i"} = $netdesc;
 787     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 788 }
 789
 790 foreach my $key (keys %$confdesc_cloudinit) {
 791     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 792 }
 793
 794 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 795 sub verify_volume_id_or_qm_path {
 796     my ($volid, $noerr) = @_;
 797
 798     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 799         return $volid;
 800     }
 801
 802     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 803     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 804     if ($@) {
 805         return undef if $noerr;
 806         die $@;
 807     }
 808     return $volid;
 809 }
 810
 811 my $drivename_hash;
 812
 813 my %drivedesc_base = (
 814     volume => { alias => 'file' },
 815     file => {
 816         type => 'string',
 817         format => 'pve-volume-id-or-qm-path',
 818         default_key => 1,
 819         format_description => 'volume',
 820         description => "The drive's backing volume.",
 821     },
 822     media => {
 823         type => 'string',
 824         enum => [qw(cdrom disk)],
 825         description => "The drive's media type.",
 826         default => 'disk',
 827         optional => 1
 828     },
 829     cyls => {
 830         type => 'integer',
 831         description => "Force the drive's physical geometry to have a specific cylinder count.",
 832         optional => 1
 833     },
 834     heads => {
 835         type => 'integer',
 836         description => "Force the drive's physical geometry to have a specific head count.",
 837         optional => 1
 838     },
 839     secs => {
 840         type => 'integer',
 841         description => "Force the drive's physical geometry to have a specific sector count.",
 842         optional => 1
 843     },
 844     trans => {
 845         type => 'string',
 846         enum => [qw(none lba auto)],
 847         description => "Force disk geometry bios translation mode.",
 848         optional => 1,
 849     },
 850     snapshot => {
 851         type => 'boolean',
 852         description => "Controls qemu's snapshot mode feature."
 853             . " If activated, changes made to the disk are temporary and will"
 854             . " be discarded when the VM is shutdown.",
 855         optional => 1,
 856     },
 857     cache => {
 858         type => 'string',
 859         enum => [qw(none writethrough writeback unsafe directsync)],
 860         description => "The drive's cache mode",
 861         optional => 1,
 862     },
 863     format => get_standard_option('pve-qm-image-format'),
 864     size => {
 865         type => 'string',
 866         format => 'disk-size',
 867         format_description => 'DiskSize',
 868         description => "Disk size. This is purely informational and has no effect.",
 869         optional => 1,
 870     },
 871     backup => {
 872         type => 'boolean',
 873         description => "Whether the drive should be included when making backups.",
 874         optional => 1,
 875     },
 876     replicate => {
 877         type => 'boolean',
 878         description => 'Whether the drive should considered for replication jobs.',
 879         optional => 1,
 880         default => 1,
 881     },
 882     rerror => {
 883         type => 'string',
 884         enum => [qw(ignore report stop)],
 885         description => 'Read error action.',
 886         optional => 1,
 887     },
 888     werror => {
 889         type => 'string',
 890         enum => [qw(enospc ignore report stop)],
 891         description => 'Write error action.',
 892         optional => 1,
 893     },
 894     aio => {
 895         type => 'string',
 896         enum => [qw(native threads)],
 897         description => 'AIO type to use.',
 898         optional => 1,
 899     },
 900     discard => {
 901         type => 'string',
 902         enum => [qw(ignore on)],
 903         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 904         optional => 1,
 905     },
 906     detect_zeroes => {
 907         type => 'boolean',
 908         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 909         optional => 1,
 910     },
 911     serial => {
 912         type => 'string',
 913         format => 'urlencoded',
 914         format_description => 'serial',
 915         maxLength => 20*3, # *3 since it's %xx url enoded
 916         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 917         optional => 1,
 918     },
 919     shared => {
 920         type => 'boolean',
 921         description => 'Mark this locally-managed volume as available on all nodes',
 922         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 923         optional => 1,
 924         default => 0,
 925     }
 926 );
 927
 928 my %iothread_fmt = ( iothread => {
 929         type => 'boolean',
 930         description => "Whether to use iothreads for this drive",
 931         optional => 1,
 932 });
 933
 934 my %model_fmt = (
 935     model => {
 936         type => 'string',
 937         format => 'urlencoded',
 938         format_description => 'model',
 939         maxLength => 40*3, # *3 since it's %xx url enoded
 940         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 941         optional => 1,
 942     },
 943 );
 944
 945 my %queues_fmt = (
 946     queues => {
 947         type => 'integer',
 948         description => "Number of queues.",
 949         minimum => 2,
 950         optional => 1
 951     }
 952 );
 953
 954 my %scsiblock_fmt = (
 955     scsiblock => {
 956         type => 'boolean',
 957         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 958         optional => 1,
 959         default => 0,
 960     },
 961 );
 962
 963 my $add_throttle_desc = sub {
 964     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 965     my $d = {
 966         type => $type,
 967         format_description => $unit,
 968         description => "Maximum $what in $longunit.",
 969         optional => 1,
 970     };
 971     $d->{minimum} = $minimum if defined($minimum);
 972     $drivedesc_base{$key} = $d;
 973 };
 974 # throughput: (leaky bucket)
 975 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 976 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 978 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 979 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 981 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 982 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 983 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 984
 985 # pools: (pool of IO before throttling starts taking effect)
 986 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 987 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 989 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 990 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 991 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 992
 993 # burst lengths
 994 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 995 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 997 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1000
1001 # legacy support
1002 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1003 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1004 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1005 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1006
1007 my $ide_fmt = {
1008     %drivedesc_base,
1009     %model_fmt,
1010 };
1011 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1012
1013 my $idedesc = {
1014     optional => 1,
1015     type => 'string', format => $ide_fmt,
1016     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1019
1020 my $scsi_fmt = {
1021     %drivedesc_base,
1022     %iothread_fmt,
1023     %queues_fmt,
1024     %scsiblock_fmt,
1025 };
1026 my $scsidesc = {
1027     optional => 1,
1028     type => 'string', format => $scsi_fmt,
1029     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1030 };
1031 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1032
1033 my $sata_fmt = {
1034     %drivedesc_base,
1035 };
1036 my $satadesc = {
1037     optional => 1,
1038     type => 'string', format => $sata_fmt,
1039     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1040 };
1041 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1042
1043 my $virtio_fmt = {
1044     %drivedesc_base,
1045     %iothread_fmt,
1046 };
1047 my $virtiodesc = {
1048     optional => 1,
1049     type => 'string', format => $virtio_fmt,
1050     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1051 };
1052 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1053
1054 my $alldrive_fmt = {
1055     %drivedesc_base,
1056     %iothread_fmt,
1057     %model_fmt,
1058     %queues_fmt,
1059     %scsiblock_fmt,
1060 };
1061
1062 my $efidisk_fmt = {
1063     volume => { alias => 'file' },
1064     file => {
1065         type => 'string',
1066         format => 'pve-volume-id-or-qm-path',
1067         default_key => 1,
1068         format_description => 'volume',
1069         description => "The drive's backing volume.",
1070     },
1071     format => get_standard_option('pve-qm-image-format'),
1072     size => {
1073         type => 'string',
1074         format => 'disk-size',
1075         format_description => 'DiskSize',
1076         description => "Disk size. This is purely informational and has no effect.",
1077         optional => 1,
1078     },
1079 };
1080
1081 my $efidisk_desc = {
1082     optional => 1,
1083     type => 'string', format => $efidisk_fmt,
1084     description => "Configure a Disk for storing EFI vars",
1085 };
1086
1087 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1088
1089 my $usb_fmt = {
1090     host => {
1091         default_key => 1,
1092         type => 'string', format => 'pve-qm-usb-device',
1093         format_description => 'HOSTUSBDEVICE|spice',
1094         description => <<EODESCR,
1095 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1096
1097  'bus-port(.port)*' (decimal numbers) or
1098  'vendor_id:product_id' (hexadeciaml numbers) or
1099  'spice'
1100
1101 You can use the 'lsusb -t' command to list existing usb devices.
1102
1103 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1104
1105 The value 'spice' can be used to add a usb redirection devices for spice.
1106 EODESCR
1107     },
1108     usb3 => {
1109         optional => 1,
1110         type => 'boolean',
1111         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1112         default => 0,
1113     },
1114 };
1115
1116 my $usbdesc = {
1117     optional => 1,
1118     type => 'string', format => $usb_fmt,
1119     description => "Configure an USB device (n is 0 to 4).",
1120 };
1121 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1122
1123 # NOTE: the match-groups of this regex are used in parse_hostpci
1124 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1125 my $hostpci_fmt = {
1126     host => {
1127         default_key => 1,
1128         type => 'string',
1129         pattern => qr/$PCIRE(;$PCIRE)*/,
1130         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1131         description => <<EODESCR,
1132 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1133 of PCI virtual functions of the host. HOSTPCIID syntax is:
1134
1135 'bus:dev.func' (hexadecimal numbers)
1136
1137 You can us the 'lspci' command to list existing PCI devices.
1138 EODESCR
1139     },
1140     rombar => {
1141         type => 'boolean',
1142         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1143         optional => 1,
1144         default => 1,
1145     },
1146     romfile => {
1147         type => 'string',
1148         pattern => '[^,;]+',
1149         format_description => 'string',
1150         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1151         optional => 1,
1152     },
1153     pcie => {
1154         type => 'boolean',
1155         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1156         optional => 1,
1157         default => 0,
1158     },
1159     'x-vga' => {
1160         type => 'boolean',
1161         description =>  "Enable vfio-vga device support.",
1162         optional => 1,
1163         default => 0,
1164     },
1165 };
1166 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1167
1168 my $hostpcidesc = {
1169         optional => 1,
1170         type => 'string', format => 'pve-qm-hostpci',
1171         description => "Map host PCI devices into guest.",
1172         verbose_description =>  <<EODESCR,
1173 Map host PCI devices into guest.
1174
1175 NOTE: This option allows direct access to host hardware. So it is no longer
1176 possible to migrate such machines - use with special care.
1177
1178 CAUTION: Experimental! User reported problems with this option.
1179 EODESCR
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1182
1183 my $serialdesc = {
1184         optional => 1,
1185         type => 'string',
1186         pattern => '(/dev/.+|socket)',
1187         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1188         verbose_description =>  <<EODESCR,
1189 Create a serial device inside the VM (n is 0 to 3), and pass through a
1190 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1191 host side (use 'qm terminal' to open a terminal connection).
1192
1193 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1194
1195 CAUTION: Experimental! User reported problems with this option.
1196 EODESCR
1197 };
1198
1199 my $paralleldesc= {
1200         optional => 1,
1201         type => 'string',
1202         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1203         description =>  "Map host parallel devices (n is 0 to 2).",
1204         verbose_description =>  <<EODESCR,
1205 Map host parallel devices (n is 0 to 2).
1206
1207 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1208
1209 CAUTION: Experimental! User reported problems with this option.
1210 EODESCR
1211 };
1212
1213 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1214     $confdesc->{"parallel$i"} = $paralleldesc;
1215 }
1216
1217 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1218     $confdesc->{"serial$i"} = $serialdesc;
1219 }
1220
1221 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1222     $confdesc->{"hostpci$i"} = $hostpcidesc;
1223 }
1224
1225 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1226     $drivename_hash->{"ide$i"} = 1;
1227     $confdesc->{"ide$i"} = $idedesc;
1228 }
1229
1230 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1231     $drivename_hash->{"sata$i"} = 1;
1232     $confdesc->{"sata$i"} = $satadesc;
1233 }
1234
1235 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1236     $drivename_hash->{"scsi$i"} = 1;
1237     $confdesc->{"scsi$i"} = $scsidesc ;
1238 }
1239
1240 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1241     $drivename_hash->{"virtio$i"} = 1;
1242     $confdesc->{"virtio$i"} = $virtiodesc;
1243 }
1244
1245 $drivename_hash->{efidisk0} = 1;
1246 $confdesc->{efidisk0} = $efidisk_desc;
1247
1248 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1249     $confdesc->{"usb$i"} = $usbdesc;
1250 }
1251
1252 my $unuseddesc = {
1253     optional => 1,
1254     type => 'string', format => 'pve-volume-id',
1255     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1256 };
1257
1258 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1259     $confdesc->{"unused$i"} = $unuseddesc;
1260 }
1261
1262 my $kvm_api_version = 0;
1263
1264 sub kvm_version {
1265
1266     return $kvm_api_version if $kvm_api_version;
1267
1268     my $fh = IO::File->new("</dev/kvm") ||
1269         return 0;
1270
1271     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1272         $kvm_api_version = $v;
1273     }
1274
1275     $fh->close();
1276
1277     return  $kvm_api_version;
1278 }
1279
1280 my $kvm_user_version;
1281
1282 sub kvm_user_version {
1283
1284     return $kvm_user_version if $kvm_user_version;
1285
1286     $kvm_user_version = 'unknown';
1287
1288     my $code = sub {
1289         my $line = shift;
1290         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1291             $kvm_user_version = $2;
1292         }
1293     };
1294
1295     eval { run_command("kvm -version", outfunc => $code); };
1296     warn $@ if $@;
1297
1298     return $kvm_user_version;
1299
1300 }
1301
1302 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1303
1304 sub valid_drive_names {
1305     # order is important - used to autoselect boot disk
1306     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1307             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1308             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1309             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1310             'efidisk0');
1311 }
1312
1313 sub is_valid_drivename {
1314     my $dev = shift;
1315
1316     return defined($drivename_hash->{$dev});
1317 }
1318
1319 sub option_exists {
1320     my $key = shift;
1321     return defined($confdesc->{$key});
1322 }
1323
1324 sub nic_models {
1325     return $nic_model_list;
1326 }
1327
1328 sub os_list_description {
1329
1330     return {
1331         other => 'Other',
1332         wxp => 'Windows XP',
1333         w2k => 'Windows 2000',
1334         w2k3 =>, 'Windows 2003',
1335         w2k8 => 'Windows 2008',
1336         wvista => 'Windows Vista',
1337         win7 => 'Windows 7',
1338         win8 => 'Windows 8/2012',
1339         win10 => 'Windows 10/2016',
1340         l24 => 'Linux 2.4',
1341         l26 => 'Linux 2.6',
1342     };
1343 }
1344
1345 my $cdrom_path;
1346
1347 sub get_cdrom_path {
1348
1349     return  $cdrom_path if $cdrom_path;
1350
1351     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1352     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1353     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1354 }
1355
1356 sub get_iso_path {
1357     my ($storecfg, $vmid, $cdrom) = @_;
1358
1359     if ($cdrom eq 'cdrom') {
1360         return get_cdrom_path();
1361     } elsif ($cdrom eq 'none') {
1362         return '';
1363     } elsif ($cdrom =~ m|^/|) {
1364         return $cdrom;
1365     } else {
1366         return PVE::Storage::path($storecfg, $cdrom);
1367     }
1368 }
1369
1370 # try to convert old style file names to volume IDs
1371 sub filename_to_volume_id {
1372     my ($vmid, $file, $media) = @_;
1373
1374      if (!($file eq 'none' || $file eq 'cdrom' ||
1375           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1376
1377         return undef if $file =~ m|/|;
1378
1379         if ($media && $media eq 'cdrom') {
1380             $file = "local:iso/$file";
1381         } else {
1382             $file = "local:$vmid/$file";
1383         }
1384     }
1385
1386     return $file;
1387 }
1388
1389 sub verify_media_type {
1390     my ($opt, $vtype, $media) = @_;
1391
1392     return if !$media;
1393
1394     my $etype;
1395     if ($media eq 'disk') {
1396         $etype = 'images';
1397     } elsif ($media eq 'cdrom') {
1398         $etype = 'iso';
1399     } else {
1400         die "internal error";
1401     }
1402
1403     return if ($vtype eq $etype);
1404
1405     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1406 }
1407
1408 sub cleanup_drive_path {
1409     my ($opt, $storecfg, $drive) = @_;
1410
1411     # try to convert filesystem paths to volume IDs
1412
1413     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1414         ($drive->{file} !~ m|^/dev/.+|) &&
1415         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1416         ($drive->{file} !~ m/^\d+$/)) {
1417         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1418         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1419         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1420         verify_media_type($opt, $vtype, $drive->{media});
1421         $drive->{file} = $volid;
1422     }
1423
1424     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1425 }
1426
1427 sub parse_hotplug_features {
1428     my ($data) = @_;
1429
1430     my $res = {};
1431
1432     return $res if $data eq '0';
1433
1434     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1435
1436     foreach my $feature (PVE::Tools::split_list($data)) {
1437         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1438             $res->{$1} = 1;
1439         } else {
1440             die "invalid hotplug feature '$feature'\n";
1441         }
1442     }
1443     return $res;
1444 }
1445
1446 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1447 sub pve_verify_hotplug_features {
1448     my ($value, $noerr) = @_;
1449
1450     return $value if parse_hotplug_features($value);
1451
1452     return undef if $noerr;
1453
1454     die "unable to parse hotplug option\n";
1455 }
1456
1457 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1458 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1459 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1460 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1461 #        [,iothread=on][,serial=serial][,model=model]
1462
1463 sub parse_drive {
1464     my ($key, $data) = @_;
1465
1466     my ($interface, $index);
1467
1468     if ($key =~ m/^([^\d]+)(\d+)$/) {
1469         $interface = $1;
1470         $index = $2;
1471     } else {
1472         return undef;
1473     }
1474
1475     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1476                                      : $confdesc->{$key}->{format};
1477     if (!$desc) {
1478         warn "invalid drive key: $key\n";
1479         return undef;
1480     }
1481     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1482     return undef if !$res;
1483     $res->{interface} = $interface;
1484     $res->{index} = $index;
1485
1486     my $error = 0;
1487     foreach my $opt (qw(bps bps_rd bps_wr)) {
1488         if (my $bps = defined(delete $res->{$opt})) {
1489             if (defined($res->{"m$opt"})) {
1490                 warn "both $opt and m$opt specified\n";
1491                 ++$error;
1492                 next;
1493             }
1494             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1495         }
1496     }
1497
1498     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1499     for my $requirement (
1500         [mbps_max => 'mbps'],
1501         [mbps_rd_max => 'mbps_rd'],
1502         [mbps_wr_max => 'mbps_wr'],
1503         [miops_max => 'miops'],
1504         [miops_rd_max => 'miops_rd'],
1505         [miops_wr_max => 'miops_wr'],
1506         [bps_max_length => 'mbps_max'],
1507         [bps_rd_max_length => 'mbps_rd_max'],
1508         [bps_wr_max_length => 'mbps_wr_max'],
1509         [iops_max_length => 'iops_max'],
1510         [iops_rd_max_length => 'iops_rd_max'],
1511         [iops_wr_max_length => 'iops_wr_max']) {
1512         my ($option, $requires) = @$requirement;
1513         if ($res->{$option} && !$res->{$requires}) {
1514             warn "$option requires $requires\n";
1515             ++$error;
1516         }
1517     }
1518
1519     return undef if $error;
1520
1521     return undef if $res->{mbps_rd} && $res->{mbps};
1522     return undef if $res->{mbps_wr} && $res->{mbps};
1523     return undef if $res->{iops_rd} && $res->{iops};
1524     return undef if $res->{iops_wr} && $res->{iops};
1525
1526     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1527         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1528         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1529         return undef if $res->{interface} eq 'virtio';
1530     }
1531
1532     if (my $size = $res->{size}) {
1533         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1534     }
1535
1536     return $res;
1537 }
1538
1539 sub print_drive {
1540     my ($vmid, $drive) = @_;
1541     my $data = { %$drive };
1542     delete $data->{$_} for qw(index interface);
1543     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1544 }
1545
1546 sub scsi_inquiry {
1547     my($fh, $noerr) = @_;
1548
1549     my $SG_IO = 0x2285;
1550     my $SG_GET_VERSION_NUM = 0x2282;
1551
1552     my $versionbuf = "\x00" x 8;
1553     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1554     if (!$ret) {
1555         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1556         return undef;
1557     }
1558     my $version = unpack("I", $versionbuf);
1559     if ($version < 30000) {
1560         die "scsi generic interface too old\n"  if !$noerr;
1561         return undef;
1562     }
1563
1564     my $buf = "\x00" x 36;
1565     my $sensebuf = "\x00" x 8;
1566     my $cmd = pack("C x3 C x1", 0x12, 36);
1567
1568     # see /usr/include/scsi/sg.h
1569     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1570
1571     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1572                       length($sensebuf), 0, length($buf), $buf,
1573                       $cmd, $sensebuf, 6000);
1574
1575     $ret = ioctl($fh, $SG_IO, $packet);
1576     if (!$ret) {
1577         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1578         return undef;
1579     }
1580
1581     my @res = unpack($sg_io_hdr_t, $packet);
1582     if ($res[17] || $res[18]) {
1583         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1584         return undef;
1585     }
1586
1587     my $res = {};
1588     (my $byte0, my $byte1, $res->{vendor},
1589      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1590
1591     $res->{removable} = $byte1 & 128 ? 1 : 0;
1592     $res->{type} = $byte0 & 31;
1593
1594     return $res;
1595 }
1596
1597 sub path_is_scsi {
1598     my ($path) = @_;
1599
1600     my $fh = IO::File->new("+<$path") || return undef;
1601     my $res = scsi_inquiry($fh, 1);
1602     close($fh);
1603
1604     return $res;
1605 }
1606
1607 sub machine_type_is_q35 {
1608     my ($conf) = @_;
1609
1610     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1611 }
1612
1613 sub print_tabletdevice_full {
1614     my ($conf) = @_;
1615
1616     my $q35 = machine_type_is_q35($conf);
1617
1618     # we use uhci for old VMs because tablet driver was buggy in older qemu
1619     my $usbbus = $q35 ? "ehci" : "uhci";
1620
1621     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1622 }
1623
1624 sub print_drivedevice_full {
1625     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1626
1627     my $device = '';
1628     my $maxdev = 0;
1629
1630     if ($drive->{interface} eq 'virtio') {
1631         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1632         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1633         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1634     } elsif ($drive->{interface} eq 'scsi') {
1635
1636         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1637         my $unit = $drive->{index} % $maxdev;
1638         my $devicetype = 'hd';
1639         my $path = '';
1640         if (drive_is_cdrom($drive)) {
1641             $devicetype = 'cd';
1642         } else {
1643             if ($drive->{file} =~ m|^/|) {
1644                 $path = $drive->{file};
1645                 if (my $info = path_is_scsi($path)) {
1646                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1647                         $devicetype = 'block';
1648                     } elsif ($info->{type} == 1) { # tape
1649                         $devicetype = 'generic';
1650                     }
1651                 }
1652             } else {
1653                  $path = PVE::Storage::path($storecfg, $drive->{file});
1654             }
1655
1656             if($path =~ m/^iscsi\:\/\//){
1657                 $devicetype = 'generic';
1658             }
1659         }
1660
1661         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1662             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1663         } else {
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         }
1666
1667     } elsif ($drive->{interface} eq 'ide'){
1668         $maxdev = 2;
1669         my $controller = int($drive->{index} / $maxdev);
1670         my $unit = $drive->{index} % $maxdev;
1671         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1672
1673         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1674         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1675             $model = URI::Escape::uri_unescape($model);
1676             $device .= ",model=$model";
1677         }
1678     } elsif ($drive->{interface} eq 'sata'){
1679         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1680         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1681         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1682     } elsif ($drive->{interface} eq 'usb') {
1683         die "implement me";
1684         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1685     } else {
1686         die "unsupported interface type";
1687     }
1688
1689     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1690
1691     return $device;
1692 }
1693
1694 sub get_initiator_name {
1695     my $initiator;
1696
1697     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1698     while (defined(my $line = <$fh>)) {
1699         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1700         $initiator = $1;
1701         last;
1702     }
1703     $fh->close();
1704
1705     return $initiator;
1706 }
1707
1708 sub print_drive_full {
1709     my ($storecfg, $vmid, $drive) = @_;
1710
1711     my $path;
1712     my $volid = $drive->{file};
1713     my $format;
1714
1715     if (drive_is_cdrom($drive)) {
1716         $path = get_iso_path($storecfg, $vmid, $volid);
1717     } else {
1718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1719         if ($storeid) {
1720             $path = PVE::Storage::path($storecfg, $volid);
1721             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1722             $format = qemu_img_format($scfg, $volname);
1723         } else {
1724             $path = $volid;
1725             $format = "raw";
1726         }
1727    }
1728
1729     my $opts = '';
1730     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1731     foreach my $o (@qemu_drive_options) {
1732         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1733     }
1734
1735     # snapshot only accepts on|off
1736     if (defined($drive->{snapshot})) {
1737         my $v = $drive->{snapshot} ? 'on' : 'off';
1738         $opts .= ",snapshot=$v";
1739     }
1740
1741     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1742         my ($dir, $qmpname) = @$type;
1743         if (my $v = $drive->{"mbps$dir"}) {
1744             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1745         }
1746         if (my $v = $drive->{"mbps${dir}_max"}) {
1747             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1748         }
1749         if (my $v = $drive->{"bps${dir}_max_length"}) {
1750             $opts .= ",throttling.bps$qmpname-max-length=$v";
1751         }
1752         if (my $v = $drive->{"iops${dir}"}) {
1753             $opts .= ",throttling.iops$qmpname=$v";
1754         }
1755         if (my $v = $drive->{"iops${dir}_max"}) {
1756             $opts .= ",throttling.iops$qmpname-max=$v";
1757         }
1758         if (my $v = $drive->{"iops${dir}_max_length"}) {
1759             $opts .= ",throttling.iops$qmpname-max-length=$v";
1760         }
1761     }
1762
1763     if (my $serial = $drive->{serial}) {
1764         $serial = URI::Escape::uri_unescape($serial);
1765         $opts .= ",serial=$serial";
1766     }
1767
1768     $opts .= ",format=$format" if $format && !$drive->{format};
1769
1770     my $cache_direct = 0;
1771
1772     if (my $cache = $drive->{cache}) {
1773         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1774     } elsif (!drive_is_cdrom($drive)) {
1775         $opts .= ",cache=none";
1776         $cache_direct = 1;
1777     }
1778
1779     # aio native works only with O_DIRECT
1780     if (!$drive->{aio}) {
1781         if($cache_direct) {
1782             $opts .= ",aio=native";
1783         } else {
1784             $opts .= ",aio=threads";
1785         }
1786     }
1787
1788     if (!drive_is_cdrom($drive)) {
1789         my $detectzeroes;
1790         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1791             $detectzeroes = 'off';
1792         } elsif ($drive->{discard}) {
1793             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1794         } else {
1795             # This used to be our default with discard not being specified:
1796             $detectzeroes = 'on';
1797         }
1798         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1799     }
1800
1801     my $pathinfo = $path ? "file=$path," : '';
1802
1803     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1804 }
1805
1806 sub print_netdevice_full {
1807     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1808
1809     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1810
1811     my $device = $net->{model};
1812     if ($net->{model} eq 'virtio') {
1813          $device = 'virtio-net-pci';
1814      };
1815
1816     my $pciaddr = print_pci_addr("$netid", $bridges);
1817     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1818     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1819         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1820         my $vectors = $net->{queues} * 2 + 2;
1821         $tmpstr .= ",vectors=$vectors,mq=on";
1822     }
1823     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1824
1825     if ($use_old_bios_files) {
1826         my $romfile;
1827         if ($device eq 'virtio-net-pci') {
1828             $romfile = 'pxe-virtio.rom';
1829         } elsif ($device eq 'e1000') {
1830             $romfile = 'pxe-e1000.rom';
1831         } elsif ($device eq 'ne2k') {
1832             $romfile = 'pxe-ne2k_pci.rom';
1833         } elsif ($device eq 'pcnet') {
1834             $romfile = 'pxe-pcnet.rom';
1835         } elsif ($device eq 'rtl8139') {
1836             $romfile = 'pxe-rtl8139.rom';
1837         }
1838         $tmpstr .= ",romfile=$romfile" if $romfile;
1839     }
1840
1841     return $tmpstr;
1842 }
1843
1844 sub print_netdev_full {
1845     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1846
1847     my $i = '';
1848     if ($netid =~ m/^net(\d+)$/) {
1849         $i = int($1);
1850     }
1851
1852     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1853
1854     my $ifname = "tap${vmid}i$i";
1855
1856     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1857     die "interface name '$ifname' is too long (max 15 character)\n"
1858         if length($ifname) >= 16;
1859
1860     my $vhostparam = '';
1861     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1862
1863     my $vmname = $conf->{name} || "vm$vmid";
1864
1865     my $netdev = "";
1866     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1867
1868     if ($net->{bridge}) {
1869         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1870     } else {
1871         $netdev = "type=user,id=$netid,hostname=$vmname";
1872     }
1873
1874     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1875
1876     return $netdev;
1877 }
1878
1879
1880 sub print_cpu_device {
1881     my ($conf, $id) = @_;
1882
1883     my $kvm = $conf->{kvm} // 1;
1884     my $cpu = $kvm ? "kvm64" : "qemu64";
1885     if (my $cputype = $conf->{cpu}) {
1886         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1887             or die "Cannot parse cpu description: $cputype\n";
1888         $cpu = $cpuconf->{cputype};
1889     }
1890
1891     my $cores = $conf->{cores} || 1;
1892
1893     my $current_core = ($id - 1) % $cores;
1894     my $current_socket = int(($id - 1 - $current_core)/$cores);
1895
1896     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1897 }
1898
1899 sub drive_is_cloudinit {
1900     my ($drive) = @_;
1901     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1902 }
1903
1904 sub drive_is_cdrom {
1905     my ($drive, $exclude_cloudinit) = @_;
1906
1907     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1908
1909     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1910
1911 }
1912
1913 sub parse_number_sets {
1914     my ($set) = @_;
1915     my $res = [];
1916     foreach my $part (split(/;/, $set)) {
1917         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1918             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1919             push @$res, [ $1, $2 ];
1920         } else {
1921             die "invalid range: $part\n";
1922         }
1923     }
1924     return $res;
1925 }
1926
1927 sub parse_numa {
1928     my ($data) = @_;
1929
1930     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1931     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1932     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1933     return $res;
1934 }
1935
1936 sub parse_hostpci {
1937     my ($value) = @_;
1938
1939     return undef if !$value;
1940
1941     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1942
1943     my @idlist = split(/;/, $res->{host});
1944     delete $res->{host};
1945     foreach my $id (@idlist) {
1946         if ($id =~ /^$PCIRE$/) {
1947             if (defined($2)) {
1948                 push @{$res->{pciid}}, { id => $1, function => $2 };
1949             } else {
1950                 my $pcidevices = lspci($1);
1951                 $res->{pciid} = $pcidevices->{$1};
1952             }
1953         } else {
1954             # should have been caught by parse_property_string already
1955             die "failed to parse PCI id: $id\n";
1956         }
1957     }
1958     return $res;
1959 }
1960
1961 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1962 sub parse_net {
1963     my ($data) = @_;
1964
1965     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1966     if ($@) {
1967         warn $@;
1968         return undef;
1969     }
1970     if (!defined($res->{macaddr})) {
1971         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1972         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1973     }
1974     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1975     return $res;
1976 }
1977
1978 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1979 sub parse_ipconfig {
1980     my ($data) = @_;
1981
1982     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1983     if ($@) {
1984         warn $@;
1985         return undef;
1986     }
1987
1988     if ($res->{gw} && !$res->{ip}) {
1989         warn 'gateway specified without specifying an IP address';
1990         return undef;
1991     }
1992     if ($res->{gw6} && !$res->{ip6}) {
1993         warn 'IPv6 gateway specified without specifying an IPv6 address';
1994         return undef;
1995     }
1996     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1997         warn 'gateway specified together with DHCP';
1998         return undef;
1999     }
2000     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2001         # gw6 + auto/dhcp
2002         warn "IPv6 gateway specified together with $res->{ip6} address";
2003         return undef;
2004     }
2005
2006     if (!$res->{ip} && !$res->{ip6}) {
2007         return { ip => 'dhcp', ip6 => 'dhcp' };
2008     }
2009
2010     return $res;
2011 }
2012
2013 sub print_net {
2014     my $net = shift;
2015
2016     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2017 }
2018
2019 sub add_random_macs {
2020     my ($settings) = @_;
2021
2022     foreach my $opt (keys %$settings) {
2023         next if $opt !~ m/^net(\d+)$/;
2024         my $net = parse_net($settings->{$opt});
2025         next if !$net;
2026         $settings->{$opt} = print_net($net);
2027     }
2028 }
2029
2030 sub vm_is_volid_owner {
2031     my ($storecfg, $vmid, $volid) = @_;
2032
2033     if ($volid !~  m|^/|) {
2034         my ($path, $owner);
2035         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2036         if ($owner && ($owner == $vmid)) {
2037             return 1;
2038         }
2039     }
2040
2041     return undef;
2042 }
2043
2044 sub split_flagged_list {
2045     my $text = shift || '';
2046     $text =~ s/[,;]/ /g;
2047     $text =~ s/^\s+//;
2048     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2049 }
2050
2051 sub join_flagged_list {
2052     my ($how, $lst) = @_;
2053     join $how, map { $lst->{$_} . $_ } keys %$lst;
2054 }
2055
2056 sub vmconfig_delete_pending_option {
2057     my ($conf, $key, $force) = @_;
2058
2059     delete $conf->{pending}->{$key};
2060     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2061     $pending_delete_hash->{$key} = $force ? '!' : '';
2062     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2063 }
2064
2065 sub vmconfig_undelete_pending_option {
2066     my ($conf, $key) = @_;
2067
2068     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2069     delete $pending_delete_hash->{$key};
2070
2071     if (%$pending_delete_hash) {
2072         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2073     } else {
2074         delete $conf->{pending}->{delete};
2075     }
2076 }
2077
2078 sub vmconfig_register_unused_drive {
2079     my ($storecfg, $vmid, $conf, $drive) = @_;
2080
2081     if (drive_is_cloudinit($drive)) {
2082         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2083         warn $@ if $@;
2084     } elsif (!drive_is_cdrom($drive)) {
2085         my $volid = $drive->{file};
2086         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2087             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2088         }
2089     }
2090 }
2091
2092 sub vmconfig_cleanup_pending {
2093     my ($conf) = @_;
2094
2095     # remove pending changes when nothing changed
2096     my $changes;
2097     foreach my $opt (keys %{$conf->{pending}}) {
2098         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2099             $changes = 1;
2100             delete $conf->{pending}->{$opt};
2101         }
2102     }
2103
2104     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2105     my $pending_delete_hash = {};
2106     while (my ($opt, $force) = each %$current_delete_hash) {
2107         if (defined($conf->{$opt})) {
2108             $pending_delete_hash->{$opt} = $force;
2109         } else {
2110             $changes = 1;
2111         }
2112     }
2113
2114     if (%$pending_delete_hash) {
2115         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2116     } else {
2117         delete $conf->{pending}->{delete};
2118     }
2119
2120     return $changes;
2121 }
2122
2123 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2124 my $smbios1_fmt = {
2125     uuid => {
2126         type => 'string',
2127         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2128         format_description => 'UUID',
2129         description => "Set SMBIOS1 UUID.",
2130         optional => 1,
2131     },
2132     version => {
2133         type => 'string',
2134         pattern => '\S+',
2135         format_description => 'string',
2136         description => "Set SMBIOS1 version.",
2137         optional => 1,
2138     },
2139     serial => {
2140         type => 'string',
2141         pattern => '\S+',
2142         format_description => 'string',
2143         description => "Set SMBIOS1 serial number.",
2144         optional => 1,
2145     },
2146     manufacturer => {
2147         type => 'string',
2148         pattern => '\S+',
2149         format_description => 'string',
2150         description => "Set SMBIOS1 manufacturer.",
2151         optional => 1,
2152     },
2153     product => {
2154         type => 'string',
2155         pattern => '\S+',
2156         format_description => 'string',
2157         description => "Set SMBIOS1 product ID.",
2158         optional => 1,
2159     },
2160     sku => {
2161         type => 'string',
2162         pattern => '\S+',
2163         format_description => 'string',
2164         description => "Set SMBIOS1 SKU string.",
2165         optional => 1,
2166     },
2167     family => {
2168         type => 'string',
2169         pattern => '\S+',
2170         format_description => 'string',
2171         description => "Set SMBIOS1 family string.",
2172         optional => 1,
2173     },
2174 };
2175
2176 sub parse_smbios1 {
2177     my ($data) = @_;
2178
2179     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2180     warn $@ if $@;
2181     return $res;
2182 }
2183
2184 sub print_smbios1 {
2185     my ($smbios1) = @_;
2186     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2187 }
2188
2189 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2190
2191 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2192 sub verify_bootdisk {
2193     my ($value, $noerr) = @_;
2194
2195     return $value if is_valid_drivename($value);
2196
2197     return undef if $noerr;
2198
2199     die "invalid boot disk '$value'\n";
2200 }
2201
2202 sub parse_watchdog {
2203     my ($value) = @_;
2204
2205     return undef if !$value;
2206
2207     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2208     warn $@ if $@;
2209     return $res;
2210 }
2211
2212 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2213 sub verify_usb_device {
2214     my ($value, $noerr) = @_;
2215
2216     return $value if parse_usb_device($value);
2217
2218     return undef if $noerr;
2219
2220     die "unable to parse usb device\n";
2221 }
2222
2223 # add JSON properties for create and set function
2224 sub json_config_properties {
2225     my $prop = shift;
2226
2227     foreach my $opt (keys %$confdesc) {
2228         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2229         $prop->{$opt} = $confdesc->{$opt};
2230     }
2231
2232     return $prop;
2233 }
2234
2235 # return copy of $confdesc_cloudinit to generate documentation
2236 sub cloudinit_config_properties {
2237
2238     return dclone($confdesc_cloudinit);
2239 }
2240
2241 sub check_type {
2242     my ($key, $value) = @_;
2243
2244     die "unknown setting '$key'\n" if !$confdesc->{$key};
2245
2246     my $type = $confdesc->{$key}->{type};
2247
2248     if (!defined($value)) {
2249         die "got undefined value\n";
2250     }
2251
2252     if ($value =~ m/[\n\r]/) {
2253         die "property contains a line feed\n";
2254     }
2255
2256     if ($type eq 'boolean') {
2257         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2258         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2259         die "type check ('boolean') failed - got '$value'\n";
2260     } elsif ($type eq 'integer') {
2261         return int($1) if $value =~ m/^(\d+)$/;
2262         die "type check ('integer') failed - got '$value'\n";
2263     } elsif ($type eq 'number') {
2264         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2265         die "type check ('number') failed - got '$value'\n";
2266     } elsif ($type eq 'string') {
2267         if (my $fmt = $confdesc->{$key}->{format}) {
2268             PVE::JSONSchema::check_format($fmt, $value);
2269             return $value;
2270         }
2271         $value =~ s/^\"(.*)\"$/$1/;
2272         return $value;
2273     } else {
2274         die "internal error"
2275     }
2276 }
2277
2278 sub check_iommu_support{
2279     #fixme : need to check IOMMU support
2280     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2281
2282     my $iommu=1;
2283     return $iommu;
2284
2285 }
2286
2287 sub touch_config {
2288     my ($vmid) = @_;
2289
2290     my $conf = PVE::QemuConfig->config_file($vmid);
2291     utime undef, undef, $conf;
2292 }
2293
2294 sub destroy_vm {
2295     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2296
2297     my $conffile = PVE::QemuConfig->config_file($vmid);
2298
2299     my $conf = PVE::QemuConfig->load_config($vmid);
2300
2301     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2302
2303     if ($conf->{template}) {
2304         # check if any base image is still used by a linked clone
2305         foreach_drive($conf, sub {
2306                 my ($ds, $drive) = @_;
2307
2308                 return if drive_is_cdrom($drive);
2309
2310                 my $volid = $drive->{file};
2311
2312                 return if !$volid || $volid =~ m|^/|;
2313
2314                 die "base volume '$volid' is still in use by linked cloned\n"
2315                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2316
2317         });
2318     }
2319
2320     # only remove disks owned by this VM
2321     foreach_drive($conf, sub {
2322         my ($ds, $drive) = @_;
2323
2324         return if drive_is_cdrom($drive, 1);
2325
2326         my $volid = $drive->{file};
2327
2328         return if !$volid || $volid =~ m|^/|;
2329
2330         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2331         return if !$path || !$owner || ($owner != $vmid);
2332
2333         eval {
2334             PVE::Storage::vdisk_free($storecfg, $volid);
2335         };
2336         warn "Could not remove disk '$volid', check manually: $@" if $@;
2337
2338     });
2339
2340     if ($keep_empty_config) {
2341         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2342     } else {
2343         unlink $conffile;
2344     }
2345
2346     # also remove unused disk
2347     eval {
2348         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2349
2350         eval {
2351             PVE::Storage::foreach_volid($dl, sub {
2352                 my ($volid, $sid, $volname, $d) = @_;
2353                 PVE::Storage::vdisk_free($storecfg, $volid);
2354             });
2355         };
2356         warn $@ if $@;
2357
2358     };
2359     warn $@ if $@;
2360 }
2361
2362 sub parse_vm_config {
2363     my ($filename, $raw) = @_;
2364
2365     return undef if !defined($raw);
2366
2367     my $res = {
2368         digest => Digest::SHA::sha1_hex($raw),
2369         snapshots => {},
2370         pending => {},
2371     };
2372
2373     $filename =~ m|/qemu-server/(\d+)\.conf$|
2374         || die "got strange filename '$filename'";
2375
2376     my $vmid = $1;
2377
2378     my $conf = $res;
2379     my $descr;
2380     my $section = '';
2381
2382     my @lines = split(/\n/, $raw);
2383     foreach my $line (@lines) {
2384         next if $line =~ m/^\s*$/;
2385
2386         if ($line =~ m/^\[PENDING\]\s*$/i) {
2387             $section = 'pending';
2388             if (defined($descr)) {
2389                 $descr =~ s/\s+$//;
2390                 $conf->{description} = $descr;
2391             }
2392             $descr = undef;
2393             $conf = $res->{$section} = {};
2394             next;
2395
2396         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2397             $section = $1;
2398             if (defined($descr)) {
2399                 $descr =~ s/\s+$//;
2400                 $conf->{description} = $descr;
2401             }
2402             $descr = undef;
2403             $conf = $res->{snapshots}->{$section} = {};
2404             next;
2405         }
2406
2407         if ($line =~ m/^\#(.*)\s*$/) {
2408             $descr = '' if !defined($descr);
2409             $descr .= PVE::Tools::decode_text($1) . "\n";
2410             next;
2411         }
2412
2413         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2414             $descr = '' if !defined($descr);
2415             $descr .= PVE::Tools::decode_text($2);
2416         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2417             $conf->{snapstate} = $1;
2418         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2419             my $key = $1;
2420             my $value = $2;
2421             $conf->{$key} = $value;
2422         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2423             my $value = $1;
2424             if ($section eq 'pending') {
2425                 $conf->{delete} = $value; # we parse this later
2426             } else {
2427                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2428             }
2429         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2430             my $key = $1;
2431             my $value = $2;
2432             eval { $value = check_type($key, $value); };
2433             if ($@) {
2434                 warn "vm $vmid - unable to parse value of '$key' - $@";
2435             } else {
2436                 $key = 'ide2' if $key eq 'cdrom';
2437                 my $fmt = $confdesc->{$key}->{format};
2438                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2439                     my $v = parse_drive($key, $value);
2440                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2441                         $v->{file} = $volid;
2442                         $value = print_drive($vmid, $v);
2443                     } else {
2444                         warn "vm $vmid - unable to parse value of '$key'\n";
2445                         next;
2446                     }
2447                 }
2448
2449                 $conf->{$key} = $value;
2450             }
2451         }
2452     }
2453
2454     if (defined($descr)) {
2455         $descr =~ s/\s+$//;
2456         $conf->{description} = $descr;
2457     }
2458     delete $res->{snapstate}; # just to be sure
2459
2460     return $res;
2461 }
2462
2463 sub write_vm_config {
2464     my ($filename, $conf) = @_;
2465
2466     delete $conf->{snapstate}; # just to be sure
2467
2468     if ($conf->{cdrom}) {
2469         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2470         $conf->{ide2} = $conf->{cdrom};
2471         delete $conf->{cdrom};
2472     }
2473
2474     # we do not use 'smp' any longer
2475     if ($conf->{sockets}) {
2476         delete $conf->{smp};
2477     } elsif ($conf->{smp}) {
2478         $conf->{sockets} = $conf->{smp};
2479         delete $conf->{cores};
2480         delete $conf->{smp};
2481     }
2482
2483     my $used_volids = {};
2484
2485     my $cleanup_config = sub {
2486         my ($cref, $pending, $snapname) = @_;
2487
2488         foreach my $key (keys %$cref) {
2489             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2490                 $key eq 'snapstate' || $key eq 'pending';
2491             my $value = $cref->{$key};
2492             if ($key eq 'delete') {
2493                 die "propertry 'delete' is only allowed in [PENDING]\n"
2494                     if !$pending;
2495                 # fixme: check syntax?
2496                 next;
2497             }
2498             eval { $value = check_type($key, $value); };
2499             die "unable to parse value of '$key' - $@" if $@;
2500
2501             $cref->{$key} = $value;
2502
2503             if (!$snapname && is_valid_drivename($key)) {
2504                 my $drive = parse_drive($key, $value);
2505                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2506             }
2507         }
2508     };
2509
2510     &$cleanup_config($conf);
2511
2512     &$cleanup_config($conf->{pending}, 1);
2513
2514     foreach my $snapname (keys %{$conf->{snapshots}}) {
2515         die "internal error" if $snapname eq 'pending';
2516         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2517     }
2518
2519     # remove 'unusedX' settings if we re-add a volume
2520     foreach my $key (keys %$conf) {
2521         my $value = $conf->{$key};
2522         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2523             delete $conf->{$key};
2524         }
2525     }
2526
2527     my $generate_raw_config = sub {
2528         my ($conf, $pending) = @_;
2529
2530         my $raw = '';
2531
2532         # add description as comment to top of file
2533         if (defined(my $descr = $conf->{description})) {
2534             if ($descr) {
2535                 foreach my $cl (split(/\n/, $descr)) {
2536                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2537                 }
2538             } else {
2539                 $raw .= "#\n" if $pending;
2540             }
2541         }
2542
2543         foreach my $key (sort keys %$conf) {
2544             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2545             $raw .= "$key: $conf->{$key}\n";
2546         }
2547         return $raw;
2548     };
2549
2550     my $raw = &$generate_raw_config($conf);
2551
2552     if (scalar(keys %{$conf->{pending}})){
2553         $raw .= "\n[PENDING]\n";
2554         $raw .= &$generate_raw_config($conf->{pending}, 1);
2555     }
2556
2557     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2558         $raw .= "\n[$snapname]\n";
2559         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2560     }
2561
2562     return $raw;
2563 }
2564
2565 sub load_defaults {
2566
2567     my $res = {};
2568
2569     # we use static defaults from our JSON schema configuration
2570     foreach my $key (keys %$confdesc) {
2571         if (defined(my $default = $confdesc->{$key}->{default})) {
2572             $res->{$key} = $default;
2573         }
2574     }
2575
2576     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2577     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2578
2579     return $res;
2580 }
2581
2582 sub config_list {
2583     my $vmlist = PVE::Cluster::get_vmlist();
2584     my $res = {};
2585     return $res if !$vmlist || !$vmlist->{ids};
2586     my $ids = $vmlist->{ids};
2587
2588     foreach my $vmid (keys %$ids) {
2589         my $d = $ids->{$vmid};
2590         next if !$d->{node} || $d->{node} ne $nodename;
2591         next if !$d->{type} || $d->{type} ne 'qemu';
2592         $res->{$vmid}->{exists} = 1;
2593     }
2594     return $res;
2595 }
2596
2597 # test if VM uses local resources (to prevent migration)
2598 sub check_local_resources {
2599     my ($conf, $noerr) = @_;
2600
2601     my $loc_res = 0;
2602
2603     $loc_res = 1 if $conf->{hostusb}; # old syntax
2604     $loc_res = 1 if $conf->{hostpci}; # old syntax
2605
2606     foreach my $k (keys %$conf) {
2607         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2608         # sockets are safe: they will recreated be on the target side post-migrate
2609         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2610         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2611     }
2612
2613     die "VM uses local resources\n" if $loc_res && !$noerr;
2614
2615     return $loc_res;
2616 }
2617
2618 # check if used storages are available on all nodes (use by migrate)
2619 sub check_storage_availability {
2620     my ($storecfg, $conf, $node) = @_;
2621
2622     foreach_drive($conf, sub {
2623         my ($ds, $drive) = @_;
2624
2625         my $volid = $drive->{file};
2626         return if !$volid;
2627
2628         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2629         return if !$sid;
2630
2631         # check if storage is available on both nodes
2632         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2633         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2634    });
2635 }
2636
2637 # list nodes where all VM images are available (used by has_feature API)
2638 sub shared_nodes {
2639     my ($conf, $storecfg) = @_;
2640
2641     my $nodelist = PVE::Cluster::get_nodelist();
2642     my $nodehash = { map { $_ => 1 } @$nodelist };
2643     my $nodename = PVE::INotify::nodename();
2644
2645     foreach_drive($conf, sub {
2646         my ($ds, $drive) = @_;
2647
2648         my $volid = $drive->{file};
2649         return if !$volid;
2650
2651         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2652         if ($storeid) {
2653             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2654             if ($scfg->{disable}) {
2655                 $nodehash = {};
2656             } elsif (my $avail = $scfg->{nodes}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     delete $nodehash->{$node} if !$avail->{$node};
2659                 }
2660             } elsif (!$scfg->{shared}) {
2661                 foreach my $node (keys %$nodehash) {
2662                     delete $nodehash->{$node} if $node ne $nodename
2663                 }
2664             }
2665         }
2666     });
2667
2668     return $nodehash
2669 }
2670
2671 sub check_cmdline {
2672     my ($pidfile, $pid) = @_;
2673
2674     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2675     if (defined($fh)) {
2676         my $line = <$fh>;
2677         $fh->close;
2678         return undef if !$line;
2679         my @param = split(/\0/, $line);
2680
2681         my $cmd = $param[0];
2682         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2683
2684         for (my $i = 0; $i < scalar (@param); $i++) {
2685             my $p = $param[$i];
2686             next if !$p;
2687             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2688                 my $p = $param[$i+1];
2689                 return 1 if $p && ($p eq $pidfile);
2690                 return undef;
2691             }
2692         }
2693     }
2694     return undef;
2695 }
2696
2697 sub check_running {
2698     my ($vmid, $nocheck, $node) = @_;
2699
2700     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2701
2702     die "unable to find configuration file for VM $vmid - no such machine\n"
2703         if !$nocheck && ! -f $filename;
2704
2705     my $pidfile = pidfile_name($vmid);
2706
2707     if (my $fd = IO::File->new("<$pidfile")) {
2708         my $st = stat($fd);
2709         my $line = <$fd>;
2710         close($fd);
2711
2712         my $mtime = $st->mtime;
2713         if ($mtime > time()) {
2714             warn "file '$filename' modified in future\n";
2715         }
2716
2717         if ($line =~ m/^(\d+)$/) {
2718             my $pid = $1;
2719             if (check_cmdline($pidfile, $pid)) {
2720                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2721                     return $pid;
2722                 }
2723             }
2724         }
2725     }
2726
2727     return undef;
2728 }
2729
2730 sub vzlist {
2731
2732     my $vzlist = config_list();
2733
2734     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2735
2736     while (defined(my $de = $fd->read)) {
2737         next if $de !~ m/^(\d+)\.pid$/;
2738         my $vmid = $1;
2739         next if !defined($vzlist->{$vmid});
2740         if (my $pid = check_running($vmid)) {
2741             $vzlist->{$vmid}->{pid} = $pid;
2742         }
2743     }
2744
2745     return $vzlist;
2746 }
2747
2748 sub disksize {
2749     my ($storecfg, $conf) = @_;
2750
2751     my $bootdisk = $conf->{bootdisk};
2752     return undef if !$bootdisk;
2753     return undef if !is_valid_drivename($bootdisk);
2754
2755     return undef if !$conf->{$bootdisk};
2756
2757     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2758     return undef if !defined($drive);
2759
2760     return undef if drive_is_cdrom($drive);
2761
2762     my $volid = $drive->{file};
2763     return undef if !$volid;
2764
2765     return $drive->{size};
2766 }
2767
2768 my $last_proc_pid_stat;
2769
2770 # get VM status information
2771 # This must be fast and should not block ($full == false)
2772 # We only query KVM using QMP if $full == true (this can be slow)
2773 sub vmstatus {
2774     my ($opt_vmid, $full) = @_;
2775
2776     my $res = {};
2777
2778     my $storecfg = PVE::Storage::config();
2779
2780     my $list = vzlist();
2781     my $defaults = load_defaults();
2782
2783     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2784
2785     my $cpucount = $cpuinfo->{cpus} || 1;
2786
2787     foreach my $vmid (keys %$list) {
2788         next if $opt_vmid && ($vmid ne $opt_vmid);
2789
2790         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2791         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2792
2793         my $d = {};
2794         $d->{pid} = $list->{$vmid}->{pid};
2795
2796         # fixme: better status?
2797         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2798
2799         my $size = disksize($storecfg, $conf);
2800         if (defined($size)) {
2801             $d->{disk} = 0; # no info available
2802             $d->{maxdisk} = $size;
2803         } else {
2804             $d->{disk} = 0;
2805             $d->{maxdisk} = 0;
2806         }
2807
2808         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2809             * ($conf->{cores} || $defaults->{cores});
2810         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2811         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2812
2813         $d->{name} = $conf->{name} || "VM $vmid";
2814         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2815             : $defaults->{memory}*(1024*1024);
2816
2817         if ($conf->{balloon}) {
2818             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2819             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2820                 : $defaults->{shares};
2821         }
2822
2823         $d->{uptime} = 0;
2824         $d->{cpu} = 0;
2825         $d->{mem} = 0;
2826
2827         $d->{netout} = 0;
2828         $d->{netin} = 0;
2829
2830         $d->{diskread} = 0;
2831         $d->{diskwrite} = 0;
2832
2833         $d->{template} = PVE::QemuConfig->is_template($conf);
2834
2835         $d->{serial} = 1 if conf_has_serial($conf);
2836
2837         $res->{$vmid} = $d;
2838     }
2839
2840     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2841     foreach my $dev (keys %$netdev) {
2842         next if $dev !~ m/^tap([1-9]\d*)i/;
2843         my $vmid = $1;
2844         my $d = $res->{$vmid};
2845         next if !$d;
2846
2847         $d->{netout} += $netdev->{$dev}->{receive};
2848         $d->{netin} += $netdev->{$dev}->{transmit};
2849
2850         if ($full) {
2851             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2852             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2853         }
2854
2855     }
2856
2857     my $ctime = gettimeofday;
2858
2859     foreach my $vmid (keys %$list) {
2860
2861         my $d = $res->{$vmid};
2862         my $pid = $d->{pid};
2863         next if !$pid;
2864
2865         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2866         next if !$pstat; # not running
2867
2868         my $used = $pstat->{utime} + $pstat->{stime};
2869
2870         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2871
2872         if ($pstat->{vsize}) {
2873             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2874         }
2875
2876         my $old = $last_proc_pid_stat->{$pid};
2877         if (!$old) {
2878             $last_proc_pid_stat->{$pid} = {
2879                 time => $ctime,
2880                 used => $used,
2881                 cpu => 0,
2882             };
2883             next;
2884         }
2885
2886         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2887
2888         if ($dtime > 1000) {
2889             my $dutime = $used -  $old->{used};
2890
2891             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2892             $last_proc_pid_stat->{$pid} = {
2893                 time => $ctime,
2894                 used => $used,
2895                 cpu => $d->{cpu},
2896             };
2897         } else {
2898             $d->{cpu} = $old->{cpu};
2899         }
2900     }
2901
2902     return $res if !$full;
2903
2904     my $qmpclient = PVE::QMPClient->new();
2905
2906     my $ballooncb = sub {
2907         my ($vmid, $resp) = @_;
2908
2909         my $info = $resp->{'return'};
2910         return if !$info->{max_mem};
2911
2912         my $d = $res->{$vmid};
2913
2914         # use memory assigned to VM
2915         $d->{maxmem} = $info->{max_mem};
2916         $d->{balloon} = $info->{actual};
2917
2918         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2919             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2920             $d->{freemem} = $info->{free_mem};
2921         }
2922
2923         $d->{ballooninfo} = $info;
2924     };
2925
2926     my $blockstatscb = sub {
2927         my ($vmid, $resp) = @_;
2928         my $data = $resp->{'return'} || [];
2929         my $totalrdbytes = 0;
2930         my $totalwrbytes = 0;
2931
2932         for my $blockstat (@$data) {
2933             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2934             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2935
2936             $blockstat->{device} =~ s/drive-//;
2937             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2938         }
2939         $res->{$vmid}->{diskread} = $totalrdbytes;
2940         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2941     };
2942
2943     my $statuscb = sub {
2944         my ($vmid, $resp) = @_;
2945
2946         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2947         # this fails if ballon driver is not loaded, so this must be
2948         # the last commnand (following command are aborted if this fails).
2949         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2950
2951         my $status = 'unknown';
2952         if (!defined($status = $resp->{'return'}->{status})) {
2953             warn "unable to get VM status\n";
2954             return;
2955         }
2956
2957         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2958     };
2959
2960     foreach my $vmid (keys %$list) {
2961         next if $opt_vmid && ($vmid ne $opt_vmid);
2962         next if !$res->{$vmid}->{pid}; # not running
2963         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2964     }
2965
2966     $qmpclient->queue_execute(undef, 2);
2967
2968     foreach my $vmid (keys %$list) {
2969         next if $opt_vmid && ($vmid ne $opt_vmid);
2970         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2971     }
2972
2973     return $res;
2974 }
2975
2976 sub foreach_drive {
2977     my ($conf, $func, @param) = @_;
2978
2979     foreach my $ds (valid_drive_names()) {
2980         next if !defined($conf->{$ds});
2981
2982         my $drive = parse_drive($ds, $conf->{$ds});
2983         next if !$drive;
2984
2985         &$func($ds, $drive, @param);
2986     }
2987 }
2988
2989 sub foreach_volid {
2990     my ($conf, $func, @param) = @_;
2991
2992     my $volhash = {};
2993
2994     my $test_volid = sub {
2995         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2996
2997         return if !$volid;
2998
2999         $volhash->{$volid}->{cdrom} //= 1;
3000         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3001
3002         $volhash->{$volid}->{replicate} //= 0;
3003         $volhash->{$volid}->{replicate} = 1 if $replicate;
3004
3005         $volhash->{$volid}->{shared} //= 0;
3006         $volhash->{$volid}->{shared} = 1 if $shared;
3007
3008         $volhash->{$volid}->{referenced_in_config} //= 0;
3009         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3010
3011         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3012             if defined($snapname);
3013     };
3014
3015     foreach_drive($conf, sub {
3016         my ($ds, $drive) = @_;
3017         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3018     });
3019
3020     foreach my $snapname (keys %{$conf->{snapshots}}) {
3021         my $snap = $conf->{snapshots}->{$snapname};
3022         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3023         foreach_drive($snap, sub {
3024             my ($ds, $drive) = @_;
3025             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3026         });
3027     }
3028
3029     foreach my $volid (keys %$volhash) {
3030         &$func($volid, $volhash->{$volid}, @param);
3031     }
3032 }
3033
3034 sub conf_has_serial {
3035     my ($conf) = @_;
3036
3037     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3038         if ($conf->{"serial$i"}) {
3039             return 1;
3040         }
3041     }
3042
3043     return 0;
3044 }
3045
3046 sub vga_conf_has_spice {
3047     my ($vga) = @_;
3048
3049     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3050
3051     return $1 || 1;
3052 }
3053
3054 sub config_to_command {
3055     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3056
3057     my $cmd = [];
3058     my $globalFlags = [];
3059     my $machineFlags = [];
3060     my $rtcFlags = [];
3061     my $cpuFlags = [];
3062     my $devices = [];
3063     my $pciaddr = '';
3064     my $bridges = {};
3065     my $kvmver = kvm_user_version();
3066     my $vernum = 0; # unknown
3067     my $ostype = $conf->{ostype};
3068     my $winversion = windows_version($ostype);
3069     my $kvm = $conf->{kvm} // 1;
3070
3071     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3072
3073     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3074         $vernum = $1*1000000+$2*1000;
3075     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3076         $vernum = $1*1000000+$2*1000+$3;
3077     }
3078
3079     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3080
3081     my $have_ovz = -f '/proc/vz/vestat';
3082
3083     my $q35 = machine_type_is_q35($conf);
3084     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3085     my $machine_type = $forcemachine || $conf->{machine};
3086     my $use_old_bios_files = undef;
3087     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3088
3089     my $cpuunits = defined($conf->{cpuunits}) ?
3090             $conf->{cpuunits} : $defaults->{cpuunits};
3091
3092     push @$cmd, '/usr/bin/kvm';
3093
3094     push @$cmd, '-id', $vmid;
3095
3096     my $vmname = $conf->{name} || "vm$vmid";
3097
3098     push @$cmd, '-name', $vmname;
3099
3100     my $use_virtio = 0;
3101
3102     my $qmpsocket = qmp_socket($vmid);
3103     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3104     push @$cmd, '-mon', "chardev=qmp,mode=control";
3105
3106
3107     push @$cmd, '-pidfile' , pidfile_name($vmid);
3108
3109     push @$cmd, '-daemonize';
3110
3111     if ($conf->{smbios1}) {
3112         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3113     }
3114
3115     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3116         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3117
3118         my $path;
3119         my $format;
3120         if (my $efidisk = $conf->{efidisk0}) {
3121             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3122             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3123             $format = $d->{format};
3124             if ($storeid) {
3125                 $path = PVE::Storage::path($storecfg, $d->{file});
3126                 if (!defined($format)) {
3127                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3128                     $format = qemu_img_format($scfg, $volname);
3129                 }
3130             } else {
3131                 $path = $d->{file};
3132                 die "efidisk format must be specified\n"
3133                     if !defined($format);
3134             }
3135         } else {
3136             warn "no efidisk configured! Using temporary efivars disk.\n";
3137             $path = "/tmp/$vmid-ovmf.fd";
3138             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3139             $format = 'raw';
3140         }
3141
3142         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3143         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3144     }
3145
3146
3147     # add usb controllers
3148     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3149     push @$devices, @usbcontrollers if @usbcontrollers;
3150     my $vga = $conf->{vga};
3151
3152     my $qxlnum = vga_conf_has_spice($vga);
3153     $vga = 'qxl' if $qxlnum;
3154
3155     if (!$vga) {
3156         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3157             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3158         } else {
3159             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3160         }
3161     }
3162
3163     # enable absolute mouse coordinates (needed by vnc)
3164     my $tablet;
3165     if (defined($conf->{tablet})) {
3166         $tablet = $conf->{tablet};
3167     } else {
3168         $tablet = $defaults->{tablet};
3169         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3170         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3171     }
3172
3173     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3174
3175     my $kvm_off = 0;
3176     my $gpu_passthrough;
3177
3178     # host pci devices
3179     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3180         my $d = parse_hostpci($conf->{"hostpci$i"});
3181         next if !$d;
3182
3183         my $pcie = $d->{pcie};
3184         if($pcie){
3185             die "q35 machine model is not enabled" if !$q35;
3186             $pciaddr = print_pcie_addr("hostpci$i");
3187         }else{
3188             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3189         }
3190
3191         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3192         my $romfile = $d->{romfile};
3193
3194         my $xvga = '';
3195         if ($d->{'x-vga'}) {
3196             $xvga = ',x-vga=on';
3197             $kvm_off = 1;
3198             $vga = 'none';
3199             $gpu_passthrough = 1;
3200
3201             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3202                 $xvga = "";
3203             }
3204         }
3205         my $pcidevices = $d->{pciid};
3206         my $multifunction = 1 if @$pcidevices > 1;
3207
3208         my $j=0;
3209         foreach my $pcidevice (@$pcidevices) {
3210
3211             my $id = "hostpci$i";
3212             $id .= ".$j" if $multifunction;
3213             my $addr = $pciaddr;
3214             $addr .= ".$j" if $multifunction;
3215             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3216
3217             if($j == 0){
3218                 $devicestr .= "$rombar$xvga";
3219                 $devicestr .= ",multifunction=on" if $multifunction;
3220                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3221             }
3222
3223             push @$devices, '-device', $devicestr;
3224             $j++;
3225         }
3226     }
3227
3228     # usb devices
3229     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3230     push @$devices, @usbdevices if @usbdevices;
3231     # serial devices
3232     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3233         if (my $path = $conf->{"serial$i"}) {
3234             if ($path eq 'socket') {
3235                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3236                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3237                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3238             } else {
3239                 die "no such serial device\n" if ! -c $path;
3240                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3241                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3242             }
3243         }
3244     }
3245
3246     # parallel devices
3247     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3248         if (my $path = $conf->{"parallel$i"}) {
3249             die "no such parallel device\n" if ! -c $path;
3250             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3251             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3252             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3253         }
3254     }
3255
3256
3257     my $sockets = 1;
3258     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3259     $sockets = $conf->{sockets} if  $conf->{sockets};
3260
3261     my $cores = $conf->{cores} || 1;
3262
3263     my $maxcpus = $sockets * $cores;
3264
3265     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3266
3267     my $allowed_vcpus = $cpuinfo->{cpus};
3268
3269     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3270         if ($allowed_vcpus < $maxcpus);
3271
3272     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3273
3274         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3275         for (my $i = 2; $i <= $vcpus; $i++)  {
3276             my $cpustr = print_cpu_device($conf,$i);
3277             push @$cmd, '-device', $cpustr;
3278         }
3279
3280     } else {
3281
3282         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3283     }
3284     push @$cmd, '-nodefaults';
3285
3286     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3287
3288     my $bootindex_hash = {};
3289     my $i = 1;
3290     foreach my $o (split(//, $bootorder)) {
3291         $bootindex_hash->{$o} = $i*100;
3292         $i++;
3293     }
3294
3295     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3296
3297     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3298
3299     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3300
3301     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3302
3303     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3304         my $socket = vnc_socket($vmid);
3305         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3306     } else {
3307         push @$cmd, '-nographic';
3308     }
3309
3310     # time drift fix
3311     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3312
3313     my $useLocaltime = $conf->{localtime};
3314
3315     if ($winversion >= 5) { # windows
3316         $useLocaltime = 1 if !defined($conf->{localtime});
3317
3318         # use time drift fix when acpi is enabled
3319         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3320             $tdf = 1 if !defined($conf->{tdf});
3321         }
3322     }
3323
3324     if ($winversion >= 6) {
3325         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3326         push @$cmd, '-no-hpet';
3327     }
3328
3329     push @$rtcFlags, 'driftfix=slew' if $tdf;
3330
3331     if (!$kvm) {
3332         push @$machineFlags, 'accel=tcg';
3333     }
3334
3335     if ($machine_type) {
3336         push @$machineFlags, "type=${machine_type}";
3337     }
3338
3339     if ($conf->{startdate}) {
3340         push @$rtcFlags, "base=$conf->{startdate}";
3341     } elsif ($useLocaltime) {
3342         push @$rtcFlags, 'base=localtime';
3343     }
3344
3345     my $cpu = $kvm ? "kvm64" : "qemu64";
3346     if (my $cputype = $conf->{cpu}) {
3347         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3348             or die "Cannot parse cpu description: $cputype\n";
3349         $cpu = $cpuconf->{cputype};
3350         $kvm_off = 1 if $cpuconf->{hidden};
3351
3352         if (defined(my $flags = $cpuconf->{flags})) {
3353             push @$cpuFlags, split(";", $flags);
3354         }
3355     }
3356
3357     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3358
3359     push @$cpuFlags , '-x2apic'
3360         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3361
3362     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3363
3364     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3365
3366     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3367
3368         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3369         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3370     }
3371
3372     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3373
3374     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3375
3376     push @$cpuFlags, 'kvm=off' if $kvm_off;
3377
3378     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3379         die "internal error"; # should not happen
3380
3381     push @$cpuFlags, "vendor=${cpu_vendor}"
3382         if $cpu_vendor ne 'default';
3383
3384     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3385
3386     push @$cmd, '-cpu', $cpu;
3387
3388     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3389
3390     push @$cmd, '-S' if $conf->{freeze};
3391
3392     # set keyboard layout
3393     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3394     push @$cmd, '-k', $kb if $kb;
3395
3396     # enable sound
3397     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3398     #push @$cmd, '-soundhw', 'es1370';
3399     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3400
3401     if($conf->{agent}) {
3402         my $qgasocket = qmp_socket($vmid, 1);
3403         my $pciaddr = print_pci_addr("qga0", $bridges);
3404         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3405         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3406         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3407     }
3408
3409     my $spice_port;
3410
3411     if ($qxlnum) {
3412         if ($qxlnum > 1) {
3413             if ($winversion){
3414                 for(my $i = 1; $i < $qxlnum; $i++){
3415                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3416                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3417                 }
3418             } else {
3419                 # assume other OS works like Linux
3420                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3421                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3422             }
3423         }
3424
3425         my $pciaddr = print_pci_addr("spice", $bridges);
3426
3427         my $nodename = PVE::INotify::nodename();
3428         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3429         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3430         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3431         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3432         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3433
3434         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3435
3436         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3437         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3438         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3439     }
3440
3441     # enable balloon by default, unless explicitly disabled
3442     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3443         $pciaddr = print_pci_addr("balloon0", $bridges);
3444         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3445     }
3446
3447     if ($conf->{watchdog}) {
3448         my $wdopts = parse_watchdog($conf->{watchdog});
3449         $pciaddr = print_pci_addr("watchdog", $bridges);
3450         my $watchdog = $wdopts->{model} || 'i6300esb';
3451         push @$devices, '-device', "$watchdog$pciaddr";
3452         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3453     }
3454
3455     my $vollist = [];
3456     my $scsicontroller = {};
3457     my $ahcicontroller = {};
3458     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3459
3460     # Add iscsi initiator name if available
3461     if (my $initiator = get_initiator_name()) {
3462         push @$devices, '-iscsi', "initiator-name=$initiator";
3463     }
3464
3465     foreach_drive($conf, sub {
3466         my ($ds, $drive) = @_;
3467
3468         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3469             push @$vollist, $drive->{file};
3470         }
3471
3472         # ignore efidisk here, already added in bios/fw handling code above
3473         return if $drive->{interface} eq 'efidisk';
3474
3475         $use_virtio = 1 if $ds =~ m/^virtio/;
3476
3477         if (drive_is_cdrom ($drive)) {
3478             if ($bootindex_hash->{d}) {
3479                 $drive->{bootindex} = $bootindex_hash->{d};
3480                 $bootindex_hash->{d} += 1;
3481             }
3482         } else {
3483             if ($bootindex_hash->{c}) {
3484                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3485                 $bootindex_hash->{c} += 1;
3486             }
3487         }
3488
3489         if($drive->{interface} eq 'virtio'){
3490            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3491         }
3492
3493         if ($drive->{interface} eq 'scsi') {
3494
3495             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3496
3497             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3498             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3499
3500             my $iothread = '';
3501             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3502                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3503                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3504             } elsif ($drive->{iothread}) {
3505                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3506             }
3507
3508             my $queues = '';
3509             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3510                 $queues = ",num_queues=$drive->{queues}";
3511             }
3512
3513             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3514             $scsicontroller->{$controller}=1;
3515         }
3516
3517         if ($drive->{interface} eq 'sata') {
3518            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3519            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3520            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3521            $ahcicontroller->{$controller}=1;
3522         }
3523
3524         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3525         push @$devices, '-drive',$drive_cmd;
3526         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3527     });
3528
3529     for (my $i = 0; $i < $MAX_NETS; $i++) {
3530          next if !$conf->{"net$i"};
3531          my $d = parse_net($conf->{"net$i"});
3532          next if !$d;
3533
3534          $use_virtio = 1 if $d->{model} eq 'virtio';
3535
3536          if ($bootindex_hash->{n}) {
3537             $d->{bootindex} = $bootindex_hash->{n};
3538             $bootindex_hash->{n} += 1;
3539          }
3540
3541          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3542          push @$devices, '-netdev', $netdevfull;
3543
3544          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3545          push @$devices, '-device', $netdevicefull;
3546     }
3547
3548     if (!$q35) {
3549         # add pci bridges
3550         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3551            $bridges->{1} = 1;
3552            $bridges->{2} = 1;
3553         }
3554
3555         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3556
3557         while (my ($k, $v) = each %$bridges) {
3558             $pciaddr = print_pci_addr("pci.$k");
3559             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3560         }
3561     }
3562
3563     # add custom args
3564     if ($conf->{args}) {
3565         my $aa = PVE::Tools::split_args($conf->{args});
3566         push @$cmd, @$aa;
3567     }
3568
3569     push @$cmd, @$devices;
3570     push @$cmd, '-rtc', join(',', @$rtcFlags)
3571         if scalar(@$rtcFlags);
3572     push @$cmd, '-machine', join(',', @$machineFlags)
3573         if scalar(@$machineFlags);
3574     push @$cmd, '-global', join(',', @$globalFlags)
3575         if scalar(@$globalFlags);
3576
3577     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3578 }
3579
3580 sub vnc_socket {
3581     my ($vmid) = @_;
3582     return "${var_run_tmpdir}/$vmid.vnc";
3583 }
3584
3585 sub spice_port {
3586     my ($vmid) = @_;
3587
3588     my $res = vm_mon_cmd($vmid, 'query-spice');
3589
3590     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3591 }
3592
3593 sub qmp_socket {
3594     my ($vmid, $qga) = @_;
3595     my $sockettype = $qga ? 'qga' : 'qmp';
3596     return "${var_run_tmpdir}/$vmid.$sockettype";
3597 }
3598
3599 sub pidfile_name {
3600     my ($vmid) = @_;
3601     return "${var_run_tmpdir}/$vmid.pid";
3602 }
3603
3604 sub vm_devices_list {
3605     my ($vmid) = @_;
3606
3607     my $res = vm_mon_cmd($vmid, 'query-pci');
3608     my $devices = {};
3609     foreach my $pcibus (@$res) {
3610         foreach my $device (@{$pcibus->{devices}}) {
3611             next if !$device->{'qdev_id'};
3612             if ($device->{'pci_bridge'}) {
3613                 $devices->{$device->{'qdev_id'}} = 1;
3614                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3615                     next if !$bridge_device->{'qdev_id'};
3616                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3617                     $devices->{$device->{'qdev_id'}}++;
3618                 }
3619             } else {
3620                 $devices->{$device->{'qdev_id'}} = 1;
3621             }
3622         }
3623     }
3624
3625     my $resblock = vm_mon_cmd($vmid, 'query-block');
3626     foreach my $block (@$resblock) {
3627         if($block->{device} =~ m/^drive-(\S+)/){
3628                 $devices->{$1} = 1;
3629         }
3630     }
3631
3632     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3633     foreach my $mice (@$resmice) {
3634         if ($mice->{name} eq 'QEMU HID Tablet') {
3635             $devices->{tablet} = 1;
3636             last;
3637         }
3638     }
3639
3640     # for usb devices there is no query-usb
3641     # but we can iterate over the entries in
3642     # qom-list path=/machine/peripheral
3643     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3644     foreach my $per (@$resperipheral) {
3645         if ($per->{name} =~ m/^usb\d+$/) {
3646             $devices->{$per->{name}} = 1;
3647         }
3648     }
3649
3650     return $devices;
3651 }
3652
3653 sub vm_deviceplug {
3654     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3655
3656     my $q35 = machine_type_is_q35($conf);
3657
3658     my $devices_list = vm_devices_list($vmid);
3659     return 1 if defined($devices_list->{$deviceid});
3660
3661     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3662
3663     if ($deviceid eq 'tablet') {
3664
3665         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3666
3667     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3668
3669         die "usb hotplug currently not reliable\n";
3670         # since we can't reliably hot unplug all added usb devices
3671         # and usb passthrough disables live migration
3672         # we disable usb hotplugging for now
3673         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3674
3675     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3676
3677         qemu_iothread_add($vmid, $deviceid, $device);
3678
3679         qemu_driveadd($storecfg, $vmid, $device);
3680         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3681
3682         qemu_deviceadd($vmid, $devicefull);
3683         eval { qemu_deviceaddverify($vmid, $deviceid); };
3684         if (my $err = $@) {
3685             eval { qemu_drivedel($vmid, $deviceid); };
3686             warn $@ if $@;
3687             die $err;
3688         }
3689
3690     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3691
3692
3693         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3694         my $pciaddr = print_pci_addr($deviceid);
3695         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3696
3697         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3698
3699         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3700             qemu_iothread_add($vmid, $deviceid, $device);
3701             $devicefull .= ",iothread=iothread-$deviceid";
3702         }
3703
3704         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3705             $devicefull .= ",num_queues=$device->{queues}";
3706         }
3707
3708         qemu_deviceadd($vmid, $devicefull);
3709         qemu_deviceaddverify($vmid, $deviceid);
3710
3711     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3712
3713         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3714         qemu_driveadd($storecfg, $vmid, $device);
3715
3716         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3717         eval { qemu_deviceadd($vmid, $devicefull); };
3718         if (my $err = $@) {
3719             eval { qemu_drivedel($vmid, $deviceid); };
3720             warn $@ if $@;
3721             die $err;
3722         }
3723
3724     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3725
3726         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3727
3728         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3729         my $use_old_bios_files = undef;
3730         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3731
3732         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3733         qemu_deviceadd($vmid, $netdevicefull);
3734         eval { qemu_deviceaddverify($vmid, $deviceid); };
3735         if (my $err = $@) {
3736             eval { qemu_netdevdel($vmid, $deviceid); };
3737             warn $@ if $@;
3738             die $err;
3739         }
3740
3741     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3742
3743         my $bridgeid = $2;
3744         my $pciaddr = print_pci_addr($deviceid);
3745         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3746
3747         qemu_deviceadd($vmid, $devicefull);
3748         qemu_deviceaddverify($vmid, $deviceid);
3749
3750     } else {
3751         die "can't hotplug device '$deviceid'\n";
3752     }
3753
3754     return 1;
3755 }
3756
3757 # fixme: this should raise exceptions on error!
3758 sub vm_deviceunplug {
3759     my ($vmid, $conf, $deviceid) = @_;
3760
3761     my $devices_list = vm_devices_list($vmid);
3762     return 1 if !defined($devices_list->{$deviceid});
3763
3764     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3765
3766     if ($deviceid eq 'tablet') {
3767
3768         qemu_devicedel($vmid, $deviceid);
3769
3770     } elsif ($deviceid =~ m/^usb\d+$/) {
3771
3772         die "usb hotplug currently not reliable\n";
3773         # when unplugging usb devices this way,
3774         # there may be remaining usb controllers/hubs
3775         # so we disable it for now
3776         qemu_devicedel($vmid, $deviceid);
3777         qemu_devicedelverify($vmid, $deviceid);
3778
3779     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3780
3781         qemu_devicedel($vmid, $deviceid);
3782         qemu_devicedelverify($vmid, $deviceid);
3783         qemu_drivedel($vmid, $deviceid);
3784         qemu_iothread_del($conf, $vmid, $deviceid);
3785
3786     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3787
3788         qemu_devicedel($vmid, $deviceid);
3789         qemu_devicedelverify($vmid, $deviceid);
3790         qemu_iothread_del($conf, $vmid, $deviceid);
3791
3792     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3793
3794         qemu_devicedel($vmid, $deviceid);
3795         qemu_drivedel($vmid, $deviceid);
3796         qemu_deletescsihw($conf, $vmid, $deviceid);
3797
3798     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3799
3800         qemu_devicedel($vmid, $deviceid);
3801         qemu_devicedelverify($vmid, $deviceid);
3802         qemu_netdevdel($vmid, $deviceid);
3803
3804     } else {
3805         die "can't unplug device '$deviceid'\n";
3806     }
3807
3808     return 1;
3809 }
3810
3811 sub qemu_deviceadd {
3812     my ($vmid, $devicefull) = @_;
3813
3814     $devicefull = "driver=".$devicefull;
3815     my %options =  split(/[=,]/, $devicefull);
3816
3817     vm_mon_cmd($vmid, "device_add" , %options);
3818 }
3819
3820 sub qemu_devicedel {
3821     my ($vmid, $deviceid) = @_;
3822
3823     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3824 }
3825
3826 sub qemu_iothread_add {
3827     my($vmid, $deviceid, $device) = @_;
3828
3829     if ($device->{iothread}) {
3830         my $iothreads = vm_iothreads_list($vmid);
3831         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3832     }
3833 }
3834
3835 sub qemu_iothread_del {
3836     my($conf, $vmid, $deviceid) = @_;
3837
3838     my $device = parse_drive($deviceid, $conf->{$deviceid});
3839     if ($device->{iothread}) {
3840         my $iothreads = vm_iothreads_list($vmid);
3841         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3842     }
3843 }
3844
3845 sub qemu_objectadd {
3846     my($vmid, $objectid, $qomtype) = @_;
3847
3848     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3849
3850     return 1;
3851 }
3852
3853 sub qemu_objectdel {
3854     my($vmid, $objectid) = @_;
3855
3856     vm_mon_cmd($vmid, "object-del", id => $objectid);
3857
3858     return 1;
3859 }
3860
3861 sub qemu_driveadd {
3862     my ($storecfg, $vmid, $device) = @_;
3863
3864     my $drive = print_drive_full($storecfg, $vmid, $device);
3865     $drive =~ s/\\/\\\\/g;
3866     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3867
3868     # If the command succeeds qemu prints: "OK"
3869     return 1 if $ret =~ m/OK/s;
3870
3871     die "adding drive failed: $ret\n";
3872 }
3873
3874 sub qemu_drivedel {
3875     my($vmid, $deviceid) = @_;
3876
3877     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3878     $ret =~ s/^\s+//;
3879
3880     return 1 if $ret eq "";
3881
3882     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3883     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3884
3885     die "deleting drive $deviceid failed : $ret\n";
3886 }
3887
3888 sub qemu_deviceaddverify {
3889     my ($vmid, $deviceid) = @_;
3890
3891     for (my $i = 0; $i <= 5; $i++) {
3892          my $devices_list = vm_devices_list($vmid);
3893          return 1 if defined($devices_list->{$deviceid});
3894          sleep 1;
3895     }
3896
3897     die "error on hotplug device '$deviceid'\n";
3898 }
3899
3900
3901 sub qemu_devicedelverify {
3902     my ($vmid, $deviceid) = @_;
3903
3904     # need to verify that the device is correctly removed as device_del
3905     # is async and empty return is not reliable
3906
3907     for (my $i = 0; $i <= 5; $i++) {
3908          my $devices_list = vm_devices_list($vmid);
3909          return 1 if !defined($devices_list->{$deviceid});
3910          sleep 1;
3911     }
3912
3913     die "error on hot-unplugging device '$deviceid'\n";
3914 }
3915
3916 sub qemu_findorcreatescsihw {
3917     my ($storecfg, $conf, $vmid, $device) = @_;
3918
3919     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3920
3921     my $scsihwid="$controller_prefix$controller";
3922     my $devices_list = vm_devices_list($vmid);
3923
3924     if(!defined($devices_list->{$scsihwid})) {
3925         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3926     }
3927
3928     return 1;
3929 }
3930
3931 sub qemu_deletescsihw {
3932     my ($conf, $vmid, $opt) = @_;
3933
3934     my $device = parse_drive($opt, $conf->{$opt});
3935
3936     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3937         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3938         return 1;
3939     }
3940
3941     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3942
3943     my $devices_list = vm_devices_list($vmid);
3944     foreach my $opt (keys %{$devices_list}) {
3945         if (PVE::QemuServer::is_valid_drivename($opt)) {
3946             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3947             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3948                 return 1;
3949             }
3950         }
3951     }
3952
3953     my $scsihwid="scsihw$controller";
3954
3955     vm_deviceunplug($vmid, $conf, $scsihwid);
3956
3957     return 1;
3958 }
3959
3960 sub qemu_add_pci_bridge {
3961     my ($storecfg, $conf, $vmid, $device) = @_;
3962
3963     my $bridges = {};
3964
3965     my $bridgeid;
3966
3967     print_pci_addr($device, $bridges);
3968
3969     while (my ($k, $v) = each %$bridges) {
3970         $bridgeid = $k;
3971     }
3972     return 1 if !defined($bridgeid) || $bridgeid < 1;
3973
3974     my $bridge = "pci.$bridgeid";
3975     my $devices_list = vm_devices_list($vmid);
3976
3977     if (!defined($devices_list->{$bridge})) {
3978         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3979     }
3980
3981     return 1;
3982 }
3983
3984 sub qemu_set_link_status {
3985     my ($vmid, $device, $up) = @_;
3986
3987     vm_mon_cmd($vmid, "set_link", name => $device,
3988                up => $up ? JSON::true : JSON::false);
3989 }
3990
3991 sub qemu_netdevadd {
3992     my ($vmid, $conf, $device, $deviceid) = @_;
3993
3994     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3995     my %options =  split(/[=,]/, $netdev);
3996
3997     vm_mon_cmd($vmid, "netdev_add",  %options);
3998     return 1;
3999 }
4000
4001 sub qemu_netdevdel {
4002     my ($vmid, $deviceid) = @_;
4003
4004     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4005 }
4006
4007 sub qemu_usb_hotplug {
4008     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4009
4010     return if !$device;
4011
4012     # remove the old one first
4013     vm_deviceunplug($vmid, $conf, $deviceid);
4014
4015     # check if xhci controller is necessary and available
4016     if ($device->{usb3}) {
4017
4018         my $devicelist = vm_devices_list($vmid);
4019
4020         if (!$devicelist->{xhci}) {
4021             my $pciaddr = print_pci_addr("xhci");
4022             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4023         }
4024     }
4025     my $d = parse_usb_device($device->{host});
4026     $d->{usb3} = $device->{usb3};
4027
4028     # add the new one
4029     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4030 }
4031
4032 sub qemu_cpu_hotplug {
4033     my ($vmid, $conf, $vcpus) = @_;
4034
4035     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4036
4037     my $sockets = 1;
4038     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4039     $sockets = $conf->{sockets} if  $conf->{sockets};
4040     my $cores = $conf->{cores} || 1;
4041     my $maxcpus = $sockets * $cores;
4042
4043     $vcpus = $maxcpus if !$vcpus;
4044
4045     die "you can't add more vcpus than maxcpus\n"
4046         if $vcpus > $maxcpus;
4047
4048     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4049
4050     if ($vcpus < $currentvcpus) {
4051
4052         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4053
4054             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4055                 qemu_devicedel($vmid, "cpu$i");
4056                 my $retry = 0;
4057                 my $currentrunningvcpus = undef;
4058                 while (1) {
4059                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4060                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4061                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4062                     $retry++;
4063                     sleep 1;
4064                 }
4065                 #update conf after each succesfull cpu unplug
4066                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4067                 PVE::QemuConfig->write_config($vmid, $conf);
4068             }
4069         } else {
4070             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4071         }
4072
4073         return;
4074     }
4075
4076     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4077     die "vcpus in running vm does not match its configuration\n"
4078         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4079
4080     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4081
4082         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4083             my $cpustr = print_cpu_device($conf, $i);
4084             qemu_deviceadd($vmid, $cpustr);
4085
4086             my $retry = 0;
4087             my $currentrunningvcpus = undef;
4088             while (1) {
4089                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4090                 last if scalar(@{$currentrunningvcpus}) == $i;
4091                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4092                 sleep 1;
4093                 $retry++;
4094             }
4095             #update conf after each succesfull cpu hotplug
4096             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4097             PVE::QemuConfig->write_config($vmid, $conf);
4098         }
4099     } else {
4100
4101         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4102             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4103         }
4104     }
4105 }
4106
4107 sub qemu_block_set_io_throttle {
4108     my ($vmid, $deviceid,
4109         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4110         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4111         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4112         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4113
4114     return if !check_running($vmid) ;
4115
4116     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4117         bps => int($bps),
4118         bps_rd => int($bps_rd),
4119         bps_wr => int($bps_wr),
4120         iops => int($iops),
4121         iops_rd => int($iops_rd),
4122         iops_wr => int($iops_wr),
4123         bps_max => int($bps_max),
4124         bps_rd_max => int($bps_rd_max),
4125         bps_wr_max => int($bps_wr_max),
4126         iops_max => int($iops_max),
4127         iops_rd_max => int($iops_rd_max),
4128         iops_wr_max => int($iops_wr_max),
4129         bps_max_length => int($bps_max_length),
4130         bps_rd_max_length => int($bps_rd_max_length),
4131         bps_wr_max_length => int($bps_wr_max_length),
4132         iops_max_length => int($iops_max_length),
4133         iops_rd_max_length => int($iops_rd_max_length),
4134         iops_wr_max_length => int($iops_wr_max_length),
4135     );
4136
4137 }
4138
4139 # old code, only used to shutdown old VM after update
4140 sub __read_avail {
4141     my ($fh, $timeout) = @_;
4142
4143     my $sel = new IO::Select;
4144     $sel->add($fh);
4145
4146     my $res = '';
4147     my $buf;
4148
4149     my @ready;
4150     while (scalar (@ready = $sel->can_read($timeout))) {
4151         my $count;
4152         if ($count = $fh->sysread($buf, 8192)) {
4153             if ($buf =~ /^(.*)\(qemu\) $/s) {
4154                 $res .= $1;
4155                 last;
4156             } else {
4157                 $res .= $buf;
4158             }
4159         } else {
4160             if (!defined($count)) {
4161                 die "$!\n";
4162             }
4163             last;
4164         }
4165     }
4166
4167     die "monitor read timeout\n" if !scalar(@ready);
4168
4169     return $res;
4170 }
4171
4172 # old code, only used to shutdown old VM after update
4173 sub vm_monitor_command {
4174     my ($vmid, $cmdstr, $nocheck) = @_;
4175
4176     my $res;
4177
4178     eval {
4179         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4180
4181         my $sname = "${var_run_tmpdir}/$vmid.mon";
4182
4183         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4184             die "unable to connect to VM $vmid socket - $!\n";
4185
4186         my $timeout = 3;
4187
4188         # hack: migrate sometime blocks the monitor (when migrate_downtime
4189         # is set)
4190         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4191             $timeout = 60*60; # 1 hour
4192         }
4193
4194         # read banner;
4195         my $data = __read_avail($sock, $timeout);
4196
4197         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4198             die "got unexpected qemu monitor banner\n";
4199         }
4200
4201         my $sel = new IO::Select;
4202         $sel->add($sock);
4203
4204         if (!scalar(my @ready = $sel->can_write($timeout))) {
4205             die "monitor write error - timeout";
4206         }
4207
4208         my $fullcmd = "$cmdstr\r";
4209
4210         # syslog('info', "VM $vmid monitor command: $cmdstr");
4211
4212         my $b;
4213         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4214             die "monitor write error - $!";
4215         }
4216
4217         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4218
4219         $timeout = 20;
4220
4221         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4222             $timeout = 60*60; # 1 hour
4223         } elsif ($cmdstr =~ m/^(eject|change)/) {
4224             $timeout = 60; # note: cdrom mount command is slow
4225         }
4226         if ($res = __read_avail($sock, $timeout)) {
4227
4228             my @lines = split("\r?\n", $res);
4229
4230             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4231
4232             $res = join("\n", @lines);
4233             $res .= "\n";
4234         }
4235     };
4236
4237     my $err = $@;
4238
4239     if ($err) {
4240         syslog("err", "VM $vmid monitor command failed - $err");
4241         die $err;
4242     }
4243
4244     return $res;
4245 }
4246
4247 sub qemu_block_resize {
4248     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4249
4250     my $running = check_running($vmid);
4251
4252     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4253
4254     return if !$running;
4255
4256     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4257
4258 }
4259
4260 sub qemu_volume_snapshot {
4261     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4262
4263     my $running = check_running($vmid);
4264
4265     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4266         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4267     } else {
4268         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4269     }
4270 }
4271
4272 sub qemu_volume_snapshot_delete {
4273     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4274
4275     my $running = check_running($vmid);
4276
4277     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4278         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4279     } else {
4280         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4281     }
4282 }
4283
4284 sub set_migration_caps {
4285     my ($vmid) = @_;
4286
4287     my $cap_ref = [];
4288
4289     my $enabled_cap = {
4290         "auto-converge" => 1,
4291         "xbzrle" => 1,
4292         "x-rdma-pin-all" => 0,
4293         "zero-blocks" => 0,
4294         "compress" => 0
4295     };
4296
4297     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4298
4299     for my $supported_capability (@$supported_capabilities) {
4300         push @$cap_ref, {
4301             capability => $supported_capability->{capability},
4302             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4303         };
4304     }
4305
4306     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4307 }
4308
4309 my $fast_plug_option = {
4310     'lock' => 1,
4311     'name' => 1,
4312     'onboot' => 1,
4313     'shares' => 1,
4314     'startup' => 1,
4315     'description' => 1,
4316     'protection' => 1,
4317     'vmstatestorage' => 1,
4318 };
4319
4320 # hotplug changes in [PENDING]
4321 # $selection hash can be used to only apply specified options, for
4322 # example: { cores => 1 } (only apply changed 'cores')
4323 # $errors ref is used to return error messages
4324 sub vmconfig_hotplug_pending {
4325     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4326
4327     my $defaults = load_defaults();
4328
4329     # commit values which do not have any impact on running VM first
4330     # Note: those option cannot raise errors, we we do not care about
4331     # $selection and always apply them.
4332
4333     my $add_error = sub {
4334         my ($opt, $msg) = @_;
4335         $errors->{$opt} = "hotplug problem - $msg";
4336     };
4337
4338     my $changes = 0;
4339     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4340         if ($fast_plug_option->{$opt}) {
4341             $conf->{$opt} = $conf->{pending}->{$opt};
4342             delete $conf->{pending}->{$opt};
4343             $changes = 1;
4344         }
4345     }
4346
4347     if ($changes) {
4348         PVE::QemuConfig->write_config($vmid, $conf);
4349         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4350     }
4351
4352     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4353
4354     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4355     while (my ($opt, $force) = each %$pending_delete_hash) {
4356         next if $selection && !$selection->{$opt};
4357         eval {
4358             if ($opt eq 'hotplug') {
4359                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4360             } elsif ($opt eq 'tablet') {
4361                 die "skip\n" if !$hotplug_features->{usb};
4362                 if ($defaults->{tablet}) {
4363                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4364                 } else {
4365                     vm_deviceunplug($vmid, $conf, $opt);
4366                 }
4367             } elsif ($opt =~ m/^usb\d+/) {
4368                 die "skip\n";
4369                 # since we cannot reliably hot unplug usb devices
4370                 # we are disabling it
4371                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4372                 vm_deviceunplug($vmid, $conf, $opt);
4373             } elsif ($opt eq 'vcpus') {
4374                 die "skip\n" if !$hotplug_features->{cpu};
4375                 qemu_cpu_hotplug($vmid, $conf, undef);
4376             } elsif ($opt eq 'balloon') {
4377                 # enable balloon device is not hotpluggable
4378                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4379             } elsif ($fast_plug_option->{$opt}) {
4380                 # do nothing
4381             } elsif ($opt =~ m/^net(\d+)$/) {
4382                 die "skip\n" if !$hotplug_features->{network};
4383                 vm_deviceunplug($vmid, $conf, $opt);
4384             } elsif (is_valid_drivename($opt)) {
4385                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4386                 vm_deviceunplug($vmid, $conf, $opt);
4387                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4388             } elsif ($opt =~ m/^memory$/) {
4389                 die "skip\n" if !$hotplug_features->{memory};
4390                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4391             } elsif ($opt eq 'cpuunits') {
4392                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4393             } elsif ($opt eq 'cpulimit') {
4394                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4395             } else {
4396                 die "skip\n";
4397             }
4398         };
4399         if (my $err = $@) {
4400             &$add_error($opt, $err) if $err ne "skip\n";
4401         } else {
4402             # save new config if hotplug was successful
4403             delete $conf->{$opt};
4404             vmconfig_undelete_pending_option($conf, $opt);
4405             PVE::QemuConfig->write_config($vmid, $conf);
4406             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4407         }
4408     }
4409
4410     my $apply_pending_cloudinit;
4411     $apply_pending_cloudinit = sub {
4412         my ($key, $value) = @_;
4413         $apply_pending_cloudinit = sub {}; # once is enough
4414
4415         my @cloudinit_opts = keys %$confdesc_cloudinit;
4416         foreach my $opt (keys %{$conf->{pending}}) {
4417             next if !grep { $_ eq $opt } @cloudinit_opts;
4418             $conf->{$opt} = delete $conf->{pending}->{$opt};
4419         }
4420
4421         my $new_conf = { %$conf };
4422         $new_conf->{$key} = $value;
4423         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4424     };
4425
4426     foreach my $opt (keys %{$conf->{pending}}) {
4427         next if $selection && !$selection->{$opt};
4428         my $value = $conf->{pending}->{$opt};
4429         eval {
4430             if ($opt eq 'hotplug') {
4431                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4432             } elsif ($opt eq 'tablet') {
4433                 die "skip\n" if !$hotplug_features->{usb};
4434                 if ($value == 1) {
4435                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4436                 } elsif ($value == 0) {
4437                     vm_deviceunplug($vmid, $conf, $opt);
4438                 }
4439             } elsif ($opt =~ m/^usb\d+$/) {
4440                 die "skip\n";
4441                 # since we cannot reliably hot unplug usb devices
4442                 # we are disabling it
4443                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4444                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4445                 die "skip\n" if !$d;
4446                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4447             } elsif ($opt eq 'vcpus') {
4448                 die "skip\n" if !$hotplug_features->{cpu};
4449                 qemu_cpu_hotplug($vmid, $conf, $value);
4450             } elsif ($opt eq 'balloon') {
4451                 # enable/disable balloning device is not hotpluggable
4452                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4453                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4454                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4455
4456                 # allow manual ballooning if shares is set to zero
4457                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4458                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4459                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4460                 }
4461             } elsif ($opt =~ m/^net(\d+)$/) {
4462                 # some changes can be done without hotplug
4463                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4464                                     $vmid, $opt, $value);
4465             } elsif (is_valid_drivename($opt)) {
4466                 # some changes can be done without hotplug
4467                 my $drive = parse_drive($opt, $value);
4468                 if (drive_is_cloudinit($drive)) {
4469                     &$apply_pending_cloudinit($opt, $value);
4470                 }
4471                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4472                                      $vmid, $opt, $value, 1);
4473             } elsif ($opt =~ m/^memory$/) { #dimms
4474                 die "skip\n" if !$hotplug_features->{memory};
4475                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4476             } elsif ($opt eq 'cpuunits') {
4477                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4478             } elsif ($opt eq 'cpulimit') {
4479                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4480                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4481             } else {
4482                 die "skip\n";  # skip non-hot-pluggable options
4483             }
4484         };
4485         if (my $err = $@) {
4486             &$add_error($opt, $err) if $err ne "skip\n";
4487         } else {
4488             # save new config if hotplug was successful
4489             $conf->{$opt} = $value;
4490             delete $conf->{pending}->{$opt};
4491             PVE::QemuConfig->write_config($vmid, $conf);
4492             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4493         }
4494     }
4495 }
4496
4497 sub try_deallocate_drive {
4498     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4499
4500     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4501         my $volid = $drive->{file};
4502         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4503             my $sid = PVE::Storage::parse_volume_id($volid);
4504             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4505
4506             # check if the disk is really unused
4507             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4508                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4509             PVE::Storage::vdisk_free($storecfg, $volid);
4510             return 1;
4511         } else {
4512             # If vm is not owner of this disk remove from config
4513             return 1;
4514         }
4515     }
4516
4517     return undef;
4518 }
4519
4520 sub vmconfig_delete_or_detach_drive {
4521     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4522
4523     my $drive = parse_drive($opt, $conf->{$opt});
4524
4525     my $rpcenv = PVE::RPCEnvironment::get();
4526     my $authuser = $rpcenv->get_user();
4527
4528     if ($force) {
4529         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4530         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4531     } else {
4532         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4533     }
4534 }
4535
4536 sub vmconfig_apply_pending {
4537     my ($vmid, $conf, $storecfg) = @_;
4538
4539     # cold plug
4540
4541     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4542     while (my ($opt, $force) = each %$pending_delete_hash) {
4543         die "internal error" if $opt =~ m/^unused/;
4544         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4545         if (!defined($conf->{$opt})) {
4546             vmconfig_undelete_pending_option($conf, $opt);
4547             PVE::QemuConfig->write_config($vmid, $conf);
4548         } elsif (is_valid_drivename($opt)) {
4549             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4550             vmconfig_undelete_pending_option($conf, $opt);
4551             delete $conf->{$opt};
4552             PVE::QemuConfig->write_config($vmid, $conf);
4553         } else {
4554             vmconfig_undelete_pending_option($conf, $opt);
4555             delete $conf->{$opt};
4556             PVE::QemuConfig->write_config($vmid, $conf);
4557         }
4558     }
4559
4560     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4561
4562     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4563         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4564
4565         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4566             # skip if nothing changed
4567         } elsif (is_valid_drivename($opt)) {
4568             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4569                 if defined($conf->{$opt});
4570             $conf->{$opt} = $conf->{pending}->{$opt};
4571         } else {
4572             $conf->{$opt} = $conf->{pending}->{$opt};
4573         }
4574
4575         delete $conf->{pending}->{$opt};
4576         PVE::QemuConfig->write_config($vmid, $conf);
4577     }
4578 }
4579
4580 my $safe_num_ne = sub {
4581     my ($a, $b) = @_;
4582
4583     return 0 if !defined($a) && !defined($b);
4584     return 1 if !defined($a);
4585     return 1 if !defined($b);
4586
4587     return $a != $b;
4588 };
4589
4590 my $safe_string_ne = sub {
4591     my ($a, $b) = @_;
4592
4593     return 0 if !defined($a) && !defined($b);
4594     return 1 if !defined($a);
4595     return 1 if !defined($b);
4596
4597     return $a ne $b;
4598 };
4599
4600 sub vmconfig_update_net {
4601     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4602
4603     my $newnet = parse_net($value);
4604
4605     if ($conf->{$opt}) {
4606         my $oldnet = parse_net($conf->{$opt});
4607
4608         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4609             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4610             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4611             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4612
4613             # for non online change, we try to hot-unplug
4614             die "skip\n" if !$hotplug;
4615             vm_deviceunplug($vmid, $conf, $opt);
4616         } else {
4617
4618             die "internal error" if $opt !~ m/net(\d+)/;
4619             my $iface = "tap${vmid}i$1";
4620
4621             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4622                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4623                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4624                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4625                 PVE::Network::tap_unplug($iface);
4626                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4627             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4628                 # Rate can be applied on its own but any change above needs to
4629                 # include the rate in tap_plug since OVS resets everything.
4630                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4631             }
4632
4633             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4634                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4635             }
4636
4637             return 1;
4638         }
4639     }
4640
4641     if ($hotplug) {
4642         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4643     } else {
4644         die "skip\n";
4645     }
4646 }
4647
4648 sub vmconfig_update_disk {
4649     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4650
4651     # fixme: do we need force?
4652
4653     my $drive = parse_drive($opt, $value);
4654
4655     if ($conf->{$opt}) {
4656
4657         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4658
4659             my $media = $drive->{media} || 'disk';
4660             my $oldmedia = $old_drive->{media} || 'disk';
4661             die "unable to change media type\n" if $media ne $oldmedia;
4662
4663             if (!drive_is_cdrom($old_drive)) {
4664
4665                 if ($drive->{file} ne $old_drive->{file}) {
4666
4667                     die "skip\n" if !$hotplug;
4668
4669                     # unplug and register as unused
4670                     vm_deviceunplug($vmid, $conf, $opt);
4671                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4672
4673                 } else {
4674                     # update existing disk
4675
4676                     # skip non hotpluggable value
4677                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4678                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4679                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4680                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4681                         die "skip\n";
4682                     }
4683
4684                     # apply throttle
4685                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4686                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4687                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4688                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4689                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4690                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4691                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4692                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4693                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4694                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4695                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4696                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4697                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4698                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4699                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4700                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4701                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4702                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4703
4704                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4705                                                    ($drive->{mbps} || 0)*1024*1024,
4706                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4707                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4708                                                    $drive->{iops} || 0,
4709                                                    $drive->{iops_rd} || 0,
4710                                                    $drive->{iops_wr} || 0,
4711                                                    ($drive->{mbps_max} || 0)*1024*1024,
4712                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4713                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4714                                                    $drive->{iops_max} || 0,
4715                                                    $drive->{iops_rd_max} || 0,
4716                                                    $drive->{iops_wr_max} || 0,
4717                                                    $drive->{bps_max_length} || 1,
4718                                                    $drive->{bps_rd_max_length} || 1,
4719                                                    $drive->{bps_wr_max_length} || 1,
4720                                                    $drive->{iops_max_length} || 1,
4721                                                    $drive->{iops_rd_max_length} || 1,
4722                                                    $drive->{iops_wr_max_length} || 1);
4723
4724                     }
4725
4726                     return 1;
4727                 }
4728
4729             } else { # cdrom
4730
4731                 if ($drive->{file} eq 'none') {
4732                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4733                     if (drive_is_cloudinit($old_drive)) {
4734                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4735                     }
4736                 } else {
4737                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4738                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4739                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4740                 }
4741
4742                 return 1;
4743             }
4744         }
4745     }
4746
4747     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4748     # hotplug new disks
4749     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4750     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4751 }
4752
4753 sub vm_start {
4754     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4755         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4756
4757     PVE::QemuConfig->lock_config($vmid, sub {
4758         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4759
4760         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4761
4762         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4763
4764         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4765
4766         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4767             vmconfig_apply_pending($vmid, $conf, $storecfg);
4768             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4769         }
4770
4771         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4772
4773         my $defaults = load_defaults();
4774
4775         # set environment variable useful inside network script
4776         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4777
4778         my $local_volumes = {};
4779
4780         if ($targetstorage) {
4781             foreach_drive($conf, sub {
4782                 my ($ds, $drive) = @_;
4783
4784                 return if drive_is_cdrom($drive);
4785
4786                 my $volid = $drive->{file};
4787
4788                 return if !$volid;
4789
4790                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4791
4792                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4793                 return if $scfg->{shared};
4794                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4795             });
4796
4797             my $format = undef;
4798
4799             foreach my $opt (sort keys %$local_volumes) {
4800
4801                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4802                 my $drive = parse_drive($opt, $conf->{$opt});
4803
4804                 #if remote storage is specified, use default format
4805                 if ($targetstorage && $targetstorage ne "1") {
4806                     $storeid = $targetstorage;
4807                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4808                     $format = $defFormat;
4809                 } else {
4810                     #else we use same format than original
4811                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4812                     $format = qemu_img_format($scfg, $volid);
4813                 }
4814
4815                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4816                 my $newdrive = $drive;
4817                 $newdrive->{format} = $format;
4818                 $newdrive->{file} = $newvolid;
4819                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4820                 $local_volumes->{$opt} = $drivestr;
4821                 #pass drive to conf for command line
4822                 $conf->{$opt} = $drivestr;
4823             }
4824         }
4825
4826         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4827
4828         my $migrate_port = 0;
4829         my $migrate_uri;
4830         if ($statefile) {
4831             if ($statefile eq 'tcp') {
4832                 my $localip = "localhost";
4833                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4834                 my $nodename = PVE::INotify::nodename();
4835
4836                 if (!defined($migration_type)) {
4837                     if (defined($datacenterconf->{migration}->{type})) {
4838                         $migration_type = $datacenterconf->{migration}->{type};
4839                     } else {
4840                         $migration_type = 'secure';
4841                     }
4842                 }
4843
4844                 if ($migration_type eq 'insecure') {
4845                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4846                     if ($migrate_network_addr) {
4847                         $localip = $migrate_network_addr;
4848                     } else {
4849                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4850                     }
4851
4852                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4853                 }
4854
4855                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4856                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4857                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4858                 push @$cmd, '-incoming', $migrate_uri;
4859                 push @$cmd, '-S';
4860
4861             } elsif ($statefile eq 'unix') {
4862                 # should be default for secure migrations as a ssh TCP forward
4863                 # tunnel is not deterministic reliable ready and fails regurarly
4864                 # to set up in time, so use UNIX socket forwards
4865                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4866                 unlink $socket_addr;
4867
4868                 $migrate_uri = "unix:$socket_addr";
4869
4870                 push @$cmd, '-incoming', $migrate_uri;
4871                 push @$cmd, '-S';
4872
4873             } else {
4874                 push @$cmd, '-loadstate', $statefile;
4875             }
4876         } elsif ($paused) {
4877             push @$cmd, '-S';
4878         }
4879
4880         # host pci devices
4881         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4882           my $d = parse_hostpci($conf->{"hostpci$i"});
4883           next if !$d;
4884           my $pcidevices = $d->{pciid};
4885           foreach my $pcidevice (@$pcidevices) {
4886                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4887
4888                 my $info = pci_device_info("0000:$pciid");
4889                 die "IOMMU not present\n" if !check_iommu_support();
4890                 die "no pci device info for device '$pciid'\n" if !$info;
4891                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4892                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4893           }
4894         }
4895
4896         PVE::Storage::activate_volumes($storecfg, $vollist);
4897
4898         if (!check_running($vmid, 1)) {
4899             eval {
4900                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4901                     outfunc => sub {}, errfunc => sub {});
4902             };
4903         }
4904
4905         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4906                                                   : $defaults->{cpuunits};
4907
4908         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4909         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4910
4911         my %properties = (
4912             Slice => 'qemu.slice',
4913             KillMode => 'none',
4914             CPUShares => $cpuunits
4915         );
4916
4917         if (my $cpulimit = $conf->{cpulimit}) {
4918             $properties{CPUQuota} = int($cpulimit * 100);
4919         }
4920         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4921
4922         if ($conf->{hugepages}) {
4923
4924             my $code = sub {
4925                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4926                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4927
4928                 PVE::QemuServer::Memory::hugepages_mount();
4929                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4930
4931                 eval  {
4932                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4933                     run_command($cmd, %run_params);
4934                 };
4935
4936                 if (my $err = $@) {
4937                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4938                     die $err;
4939                 }
4940
4941                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4942             };
4943             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4944
4945         } else {
4946             eval  {
4947                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4948                 run_command($cmd, %run_params);
4949             };
4950         }
4951
4952         if (my $err = $@) {
4953             # deactivate volumes if start fails
4954             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4955             die "start failed: $err";
4956         }
4957
4958         print "migration listens on $migrate_uri\n" if $migrate_uri;
4959
4960         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4961             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4962             warn $@ if $@;
4963         }
4964
4965         #start nbd server for storage migration
4966         if ($targetstorage) {
4967             my $nodename = PVE::INotify::nodename();
4968             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4969             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4970             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4971             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4972
4973             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4974
4975             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4976
4977             foreach my $opt (sort keys %$local_volumes) {
4978                 my $volid = $local_volumes->{$opt};
4979                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4980                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4981                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4982             }
4983         }
4984
4985         if ($migratedfrom) {
4986             eval {
4987                 set_migration_caps($vmid);
4988             };
4989             warn $@ if $@;
4990
4991             if ($spice_port) {
4992                 print "spice listens on port $spice_port\n";
4993                 if ($spice_ticket) {
4994                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4995                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4996                 }
4997             }
4998
4999         } else {
5000             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
5001                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5002                     if $conf->{balloon};
5003             }
5004
5005             foreach my $opt (keys %$conf) {
5006                 next if $opt !~  m/^net\d+$/;
5007                 my $nicconf = parse_net($conf->{$opt});
5008                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5009             }
5010         }
5011
5012         vm_mon_cmd_nocheck($vmid, 'qom-set',
5013                     path => "machine/peripheral/balloon0",
5014                     property => "guest-stats-polling-interval",
5015                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5016
5017     });
5018 }
5019
5020 sub vm_mon_cmd {
5021     my ($vmid, $execute, %params) = @_;
5022
5023     my $cmd = { execute => $execute, arguments => \%params };
5024     vm_qmp_command($vmid, $cmd);
5025 }
5026
5027 sub vm_mon_cmd_nocheck {
5028     my ($vmid, $execute, %params) = @_;
5029
5030     my $cmd = { execute => $execute, arguments => \%params };
5031     vm_qmp_command($vmid, $cmd, 1);
5032 }
5033
5034 sub vm_qmp_command {
5035     my ($vmid, $cmd, $nocheck) = @_;
5036
5037     my $res;
5038
5039     my $timeout;
5040     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5041         $timeout = $cmd->{arguments}->{timeout};
5042         delete $cmd->{arguments}->{timeout};
5043     }
5044
5045     eval {
5046         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5047         my $sname = qmp_socket($vmid);
5048         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5049             my $qmpclient = PVE::QMPClient->new();
5050
5051             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5052         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
5053             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
5054                 if scalar(%{$cmd->{arguments}});
5055             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
5056         } else {
5057             die "unable to open monitor socket\n";
5058         }
5059     };
5060     if (my $err = $@) {
5061         syslog("err", "VM $vmid qmp command failed - $err");
5062         die $err;
5063     }
5064
5065     return $res;
5066 }
5067
5068 sub vm_human_monitor_command {
5069     my ($vmid, $cmdline) = @_;
5070
5071     my $res;
5072
5073     my $cmd = {
5074         execute => 'human-monitor-command',
5075         arguments => { 'command-line' => $cmdline},
5076     };
5077
5078     return vm_qmp_command($vmid, $cmd);
5079 }
5080
5081 sub vm_commandline {
5082     my ($storecfg, $vmid) = @_;
5083
5084     my $conf = PVE::QemuConfig->load_config($vmid);
5085
5086     my $defaults = load_defaults();
5087
5088     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5089
5090     return PVE::Tools::cmd2string($cmd);
5091 }
5092
5093 sub vm_reset {
5094     my ($vmid, $skiplock) = @_;
5095
5096     PVE::QemuConfig->lock_config($vmid, sub {
5097
5098         my $conf = PVE::QemuConfig->load_config($vmid);
5099
5100         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5101
5102         vm_mon_cmd($vmid, "system_reset");
5103     });
5104 }
5105
5106 sub get_vm_volumes {
5107     my ($conf) = @_;
5108
5109     my $vollist = [];
5110     foreach_volid($conf, sub {
5111         my ($volid, $attr) = @_;
5112
5113         return if $volid =~ m|^/|;
5114
5115         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5116         return if !$sid;
5117
5118         push @$vollist, $volid;
5119     });
5120
5121     return $vollist;
5122 }
5123
5124 sub vm_stop_cleanup {
5125     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5126
5127     eval {
5128
5129         if (!$keepActive) {
5130             my $vollist = get_vm_volumes($conf);
5131             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5132         }
5133
5134         foreach my $ext (qw(mon qmp pid vnc qga)) {
5135             unlink "/var/run/qemu-server/${vmid}.$ext";
5136         }
5137
5138         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5139     };
5140     warn $@ if $@; # avoid errors - just warn
5141 }
5142
5143 # Note: use $nockeck to skip tests if VM configuration file exists.
5144 # We need that when migration VMs to other nodes (files already moved)
5145 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5146 sub vm_stop {
5147     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5148
5149     $force = 1 if !defined($force) && !$shutdown;
5150
5151     if ($migratedfrom){
5152         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5153         kill 15, $pid if $pid;
5154         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5155         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5156         return;
5157     }
5158
5159     PVE::QemuConfig->lock_config($vmid, sub {
5160
5161         my $pid = check_running($vmid, $nocheck);
5162         return if !$pid;
5163
5164         my $conf;
5165         if (!$nocheck) {
5166             $conf = PVE::QemuConfig->load_config($vmid);
5167             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5168             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5169                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5170                 $timeout = $opts->{down} if $opts->{down};
5171             }
5172         }
5173
5174         $timeout = 60 if !defined($timeout);
5175
5176         eval {
5177             if ($shutdown) {
5178                 if (defined($conf) && $conf->{agent}) {
5179                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5180                 } else {
5181                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5182                 }
5183             } else {
5184                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5185             }
5186         };
5187         my $err = $@;
5188
5189         if (!$err) {
5190             my $count = 0;
5191             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5192                 $count++;
5193                 sleep 1;
5194             }
5195
5196             if ($count >= $timeout) {
5197                 if ($force) {
5198                     warn "VM still running - terminating now with SIGTERM\n";
5199                     kill 15, $pid;
5200                 } else {
5201                     die "VM quit/powerdown failed - got timeout\n";
5202                 }
5203             } else {
5204                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5205                 return;
5206             }
5207         } else {
5208             if ($force) {
5209                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5210                 kill 15, $pid;
5211             } else {
5212                 die "VM quit/powerdown failed\n";
5213             }
5214         }
5215
5216         # wait again
5217         $timeout = 10;
5218
5219         my $count = 0;
5220         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5221             $count++;
5222             sleep 1;
5223         }
5224
5225         if ($count >= $timeout) {
5226             warn "VM still running - terminating now with SIGKILL\n";
5227             kill 9, $pid;
5228             sleep 1;
5229         }
5230
5231         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5232    });
5233 }
5234
5235 sub vm_suspend {
5236     my ($vmid, $skiplock) = @_;
5237
5238     PVE::QemuConfig->lock_config($vmid, sub {
5239
5240         my $conf = PVE::QemuConfig->load_config($vmid);
5241
5242         PVE::QemuConfig->check_lock($conf)
5243             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5244
5245         vm_mon_cmd($vmid, "stop");
5246     });
5247 }
5248
5249 sub vm_resume {
5250     my ($vmid, $skiplock, $nocheck) = @_;
5251
5252     PVE::QemuConfig->lock_config($vmid, sub {
5253
5254         if (!$nocheck) {
5255
5256             my $conf = PVE::QemuConfig->load_config($vmid);
5257
5258             PVE::QemuConfig->check_lock($conf)
5259                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5260
5261             vm_mon_cmd($vmid, "cont");
5262
5263         } else {
5264             vm_mon_cmd_nocheck($vmid, "cont");
5265         }
5266     });
5267 }
5268
5269 sub vm_sendkey {
5270     my ($vmid, $skiplock, $key) = @_;
5271
5272     PVE::QemuConfig->lock_config($vmid, sub {
5273
5274         my $conf = PVE::QemuConfig->load_config($vmid);
5275
5276         # there is no qmp command, so we use the human monitor command
5277         vm_human_monitor_command($vmid, "sendkey $key");
5278     });
5279 }
5280
5281 sub vm_destroy {
5282     my ($storecfg, $vmid, $skiplock) = @_;
5283
5284     PVE::QemuConfig->lock_config($vmid, sub {
5285
5286         my $conf = PVE::QemuConfig->load_config($vmid);
5287
5288         if (!check_running($vmid)) {
5289             destroy_vm($storecfg, $vmid, undef, $skiplock);
5290         } else {
5291             die "VM $vmid is running - destroy failed\n";
5292         }
5293     });
5294 }
5295
5296 # pci helpers
5297
5298 sub file_write {
5299     my ($filename, $buf) = @_;
5300
5301     my $fh = IO::File->new($filename, "w");
5302     return undef if !$fh;
5303
5304     my $res = print $fh $buf;
5305
5306     $fh->close();
5307
5308     return $res;
5309 }
5310
5311 sub pci_device_info {
5312     my ($name) = @_;
5313
5314     my $res;
5315
5316     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5317     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5318
5319     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5320     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5321
5322     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5323     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5324
5325     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5326     return undef if !defined($product) || $product !~ s/^0x//;
5327
5328     $res = {
5329         name => $name,
5330         vendor => $vendor,
5331         product => $product,
5332         domain => $domain,
5333         bus => $bus,
5334         slot => $slot,
5335         func => $func,
5336         irq => $irq,
5337         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5338     };
5339
5340     return $res;
5341 }
5342
5343 sub pci_dev_reset {
5344     my ($dev) = @_;
5345
5346     my $name = $dev->{name};
5347
5348     my $fn = "$pcisysfs/devices/$name/reset";
5349
5350     return file_write($fn, "1");
5351 }
5352
5353 sub pci_dev_bind_to_vfio {
5354     my ($dev) = @_;
5355
5356     my $name = $dev->{name};
5357
5358     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5359
5360     if (!-d $vfio_basedir) {
5361         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5362     }
5363     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5364
5365     my $testdir = "$vfio_basedir/$name";
5366     return 1 if -d $testdir;
5367
5368     my $data = "$dev->{vendor} $dev->{product}";
5369     return undef if !file_write("$vfio_basedir/new_id", $data);
5370
5371     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5372     if (!file_write($fn, $name)) {
5373         return undef if -f $fn;
5374     }
5375
5376     $fn = "$vfio_basedir/bind";
5377     if (! -d $testdir) {
5378         return undef if !file_write($fn, $name);
5379     }
5380
5381     return -d $testdir;
5382 }
5383
5384 sub pci_dev_group_bind_to_vfio {
5385     my ($pciid) = @_;
5386
5387     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5388
5389     if (!-d $vfio_basedir) {
5390         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5391     }
5392     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5393
5394     # get IOMMU group devices
5395     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5396       my @devs = grep /^0000:/, readdir($D);
5397     closedir($D);
5398
5399     foreach my $pciid (@devs) {
5400         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5401
5402         # pci bridges, switches or root ports are not supported
5403         # they have a pci_bus subdirectory so skip them
5404         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5405
5406         my $info = pci_device_info($1);
5407         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5408     }
5409
5410     return 1;
5411 }
5412
5413 # vzdump restore implementaion
5414
5415 sub tar_archive_read_firstfile {
5416     my $archive = shift;
5417
5418     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5419
5420     # try to detect archive type first
5421     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5422         die "unable to open file '$archive'\n";
5423     my $firstfile = <$fh>;
5424     kill 15, $pid;
5425     close $fh;
5426
5427     die "ERROR: archive contaions no data\n" if !$firstfile;
5428     chomp $firstfile;
5429
5430     return $firstfile;
5431 }
5432
5433 sub tar_restore_cleanup {
5434     my ($storecfg, $statfile) = @_;
5435
5436     print STDERR "starting cleanup\n";
5437
5438     if (my $fd = IO::File->new($statfile, "r")) {
5439         while (defined(my $line = <$fd>)) {
5440             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5441                 my $volid = $2;
5442                 eval {
5443                     if ($volid =~ m|^/|) {
5444                         unlink $volid || die 'unlink failed\n';
5445                     } else {
5446                         PVE::Storage::vdisk_free($storecfg, $volid);
5447                     }
5448                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5449                 };
5450                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5451             } else {
5452                 print STDERR "unable to parse line in statfile - $line";
5453             }
5454         }
5455         $fd->close();
5456     }
5457 }
5458
5459 sub restore_archive {
5460     my ($archive, $vmid, $user, $opts) = @_;
5461
5462     my $format = $opts->{format};
5463     my $comp;
5464
5465     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5466         $format = 'tar' if !$format;
5467         $comp = 'gzip';
5468     } elsif ($archive =~ m/\.tar$/) {
5469         $format = 'tar' if !$format;
5470     } elsif ($archive =~ m/.tar.lzo$/) {
5471         $format = 'tar' if !$format;
5472         $comp = 'lzop';
5473     } elsif ($archive =~ m/\.vma$/) {
5474         $format = 'vma' if !$format;
5475     } elsif ($archive =~ m/\.vma\.gz$/) {
5476         $format = 'vma' if !$format;
5477         $comp = 'gzip';
5478     } elsif ($archive =~ m/\.vma\.lzo$/) {
5479         $format = 'vma' if !$format;
5480         $comp = 'lzop';
5481     } else {
5482         $format = 'vma' if !$format; # default
5483     }
5484
5485     # try to detect archive format
5486     if ($format eq 'tar') {
5487         return restore_tar_archive($archive, $vmid, $user, $opts);
5488     } else {
5489         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5490     }
5491 }
5492
5493 sub restore_update_config_line {
5494     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5495
5496     return if $line =~ m/^\#qmdump\#/;
5497     return if $line =~ m/^\#vzdump\#/;
5498     return if $line =~ m/^lock:/;
5499     return if $line =~ m/^unused\d+:/;
5500     return if $line =~ m/^parent:/;
5501     return if $line =~ m/^template:/; # restored VM is never a template
5502
5503     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5504     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5505         # try to convert old 1.X settings
5506         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5507         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5508             my ($model, $macaddr) = split(/\=/, $devconfig);
5509             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5510             my $net = {
5511                 model => $model,
5512                 bridge => "vmbr$ind",
5513                 macaddr => $macaddr,
5514             };
5515             my $netstr = print_net($net);
5516
5517             print $outfd "net$cookie->{netcount}: $netstr\n";
5518             $cookie->{netcount}++;
5519         }
5520     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5521         my ($id, $netstr) = ($1, $2);
5522         my $net = parse_net($netstr);
5523         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5524         $netstr = print_net($net);
5525         print $outfd "$id: $netstr\n";
5526     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5527         my $virtdev = $1;
5528         my $value = $3;
5529         my $di = parse_drive($virtdev, $value);
5530         if (defined($di->{backup}) && !$di->{backup}) {
5531             print $outfd "#$line";
5532         } elsif ($map->{$virtdev}) {
5533             delete $di->{format}; # format can change on restore
5534             $di->{file} = $map->{$virtdev};
5535             $value = print_drive($vmid, $di);
5536             print $outfd "$virtdev: $value\n";
5537         } else {
5538             print $outfd $line;
5539         }
5540     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5541         my ($uuid, $uuid_str);
5542         UUID::generate($uuid);
5543         UUID::unparse($uuid, $uuid_str);
5544         my $smbios1 = parse_smbios1($2);
5545         $smbios1->{uuid} = $uuid_str;
5546         print $outfd $1.print_smbios1($smbios1)."\n";
5547     } else {
5548         print $outfd $line;
5549     }
5550 }
5551
5552 sub scan_volids {
5553     my ($cfg, $vmid) = @_;
5554
5555     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5556
5557     my $volid_hash = {};
5558     foreach my $storeid (keys %$info) {
5559         foreach my $item (@{$info->{$storeid}}) {
5560             next if !($item->{volid} && $item->{size});
5561             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5562             $volid_hash->{$item->{volid}} = $item;
5563         }
5564     }
5565
5566     return $volid_hash;
5567 }
5568
5569 sub is_volume_in_use {
5570     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5571
5572     my $path = PVE::Storage::path($storecfg, $volid);
5573
5574     my $scan_config = sub {
5575         my ($cref, $snapname) = @_;
5576
5577         foreach my $key (keys %$cref) {
5578             my $value = $cref->{$key};
5579             if (is_valid_drivename($key)) {
5580                 next if $skip_drive && $key eq $skip_drive;
5581                 my $drive = parse_drive($key, $value);
5582                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5583                 return 1 if $volid eq $drive->{file};
5584                 if ($drive->{file} =~ m!^/!) {
5585                     return 1 if $drive->{file} eq $path;
5586                 } else {
5587                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5588                     next if !$storeid;
5589                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5590                     next if !$scfg;
5591                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5592                 }
5593             }
5594         }
5595
5596         return 0;
5597     };
5598
5599     return 1 if &$scan_config($conf);
5600
5601     undef $skip_drive;
5602
5603     foreach my $snapname (keys %{$conf->{snapshots}}) {
5604         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5605     }
5606
5607     return 0;
5608 }
5609
5610 sub update_disksize {
5611     my ($vmid, $conf, $volid_hash) = @_;
5612
5613     my $changes;
5614
5615     # used and unused disks
5616     my $referenced = {};
5617
5618     # Note: it is allowed to define multiple storages with same path (alias), so
5619     # we need to check both 'volid' and real 'path' (two different volid can point
5620     # to the same path).
5621
5622     my $referencedpath = {};
5623
5624     # update size info
5625     foreach my $opt (keys %$conf) {
5626         if (is_valid_drivename($opt)) {
5627             my $drive = parse_drive($opt, $conf->{$opt});
5628             my $volid = $drive->{file};
5629             next if !$volid;
5630
5631             $referenced->{$volid} = 1;
5632             if ($volid_hash->{$volid} &&
5633                 (my $path = $volid_hash->{$volid}->{path})) {
5634                 $referencedpath->{$path} = 1;
5635             }
5636
5637             next if drive_is_cdrom($drive);
5638             next if !$volid_hash->{$volid};
5639
5640             $drive->{size} = $volid_hash->{$volid}->{size};
5641             my $new = print_drive($vmid, $drive);
5642             if ($new ne $conf->{$opt}) {
5643                 $changes = 1;
5644                 $conf->{$opt} = $new;
5645             }
5646         }
5647     }
5648
5649     # remove 'unusedX' entry if volume is used
5650     foreach my $opt (keys %$conf) {
5651         next if $opt !~ m/^unused\d+$/;
5652         my $volid = $conf->{$opt};
5653         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5654         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5655             $changes = 1;
5656             delete $conf->{$opt};
5657         }
5658
5659         $referenced->{$volid} = 1;
5660         $referencedpath->{$path} = 1 if $path;
5661     }
5662
5663     foreach my $volid (sort keys %$volid_hash) {
5664         next if $volid =~ m/vm-$vmid-state-/;
5665         next if $referenced->{$volid};
5666         my $path = $volid_hash->{$volid}->{path};
5667         next if !$path; # just to be sure
5668         next if $referencedpath->{$path};
5669         $changes = 1;
5670         PVE::QemuConfig->add_unused_volume($conf, $volid);
5671         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5672     }
5673
5674     return $changes;
5675 }
5676
5677 sub rescan {
5678     my ($vmid, $nolock) = @_;
5679
5680     my $cfg = PVE::Storage::config();
5681
5682     my $volid_hash = scan_volids($cfg, $vmid);
5683
5684     my $updatefn =  sub {
5685         my ($vmid) = @_;
5686
5687         my $conf = PVE::QemuConfig->load_config($vmid);
5688
5689         PVE::QemuConfig->check_lock($conf);
5690
5691         my $vm_volids = {};
5692         foreach my $volid (keys %$volid_hash) {
5693             my $info = $volid_hash->{$volid};
5694             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5695         }
5696
5697         my $changes = update_disksize($vmid, $conf, $vm_volids);
5698
5699         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5700     };
5701
5702     if (defined($vmid)) {
5703         if ($nolock) {
5704             &$updatefn($vmid);
5705         } else {
5706             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5707         }
5708     } else {
5709         my $vmlist = config_list();
5710         foreach my $vmid (keys %$vmlist) {
5711             if ($nolock) {
5712                 &$updatefn($vmid);
5713             } else {
5714                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5715             }
5716         }
5717     }
5718 }
5719
5720 sub restore_vma_archive {
5721     my ($archive, $vmid, $user, $opts, $comp) = @_;
5722
5723     my $input = $archive eq '-' ? "<&STDIN" : undef;
5724     my $readfrom = $archive;
5725
5726     my $uncomp = '';
5727     if ($comp) {
5728         $readfrom = '-';
5729         my $qarchive = PVE::Tools::shellquote($archive);
5730         if ($comp eq 'gzip') {
5731             $uncomp = "zcat $qarchive|";
5732         } elsif ($comp eq 'lzop') {
5733             $uncomp = "lzop -d -c $qarchive|";
5734         } else {
5735             die "unknown compression method '$comp'\n";
5736         }
5737
5738     }
5739
5740     my $tmpdir = "/var/tmp/vzdumptmp$$";
5741     rmtree $tmpdir;
5742
5743     # disable interrupts (always do cleanups)
5744     local $SIG{INT} =
5745         local $SIG{TERM} =
5746         local $SIG{QUIT} =
5747         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5748
5749     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5750     POSIX::mkfifo($mapfifo, 0600);
5751     my $fifofh;
5752
5753     my $openfifo = sub {
5754         open($fifofh, '>', $mapfifo) || die $!;
5755     };
5756
5757     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5758
5759     my $oldtimeout;
5760     my $timeout = 5;
5761
5762     my $devinfo = {};
5763
5764     my $rpcenv = PVE::RPCEnvironment::get();
5765
5766     my $conffile = PVE::QemuConfig->config_file($vmid);
5767     my $tmpfn = "$conffile.$$.tmp";
5768
5769     # Note: $oldconf is undef if VM does not exists
5770     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5771     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5772
5773     my $print_devmap = sub {
5774         my $virtdev_hash = {};
5775
5776         my $cfgfn = "$tmpdir/qemu-server.conf";
5777
5778         # we can read the config - that is already extracted
5779         my $fh = IO::File->new($cfgfn, "r") ||
5780             "unable to read qemu-server.conf - $!\n";
5781
5782         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5783         if (-f $fwcfgfn) {
5784             my $pve_firewall_dir = '/etc/pve/firewall';
5785             mkdir $pve_firewall_dir; # make sure the dir exists
5786             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5787         }
5788
5789         while (defined(my $line = <$fh>)) {
5790             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5791                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5792                 die "archive does not contain data for drive '$virtdev'\n"
5793                     if !$devinfo->{$devname};
5794                 if (defined($opts->{storage})) {
5795                     $storeid = $opts->{storage} || 'local';
5796                 } elsif (!$storeid) {
5797                     $storeid = 'local';
5798                 }
5799                 $format = 'raw' if !$format;
5800                 $devinfo->{$devname}->{devname} = $devname;
5801                 $devinfo->{$devname}->{virtdev} = $virtdev;
5802                 $devinfo->{$devname}->{format} = $format;
5803                 $devinfo->{$devname}->{storeid} = $storeid;
5804
5805                 # check permission on storage
5806                 my $pool = $opts->{pool}; # todo: do we need that?
5807                 if ($user ne 'root@pam') {
5808                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5809                 }
5810
5811                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5812             }
5813         }
5814
5815         foreach my $devname (keys %$devinfo) {
5816             die "found no device mapping information for device '$devname'\n"
5817                 if !$devinfo->{$devname}->{virtdev};
5818         }
5819
5820         my $cfg = PVE::Storage::config();
5821
5822         # create empty/temp config
5823         if ($oldconf) {
5824             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5825             foreach_drive($oldconf, sub {
5826                 my ($ds, $drive) = @_;
5827
5828                 return if drive_is_cdrom($drive);
5829
5830                 my $volid = $drive->{file};
5831
5832                 return if !$volid || $volid =~ m|^/|;
5833
5834                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5835                 return if !$path || !$owner || ($owner != $vmid);
5836
5837                 # Note: only delete disk we want to restore
5838                 # other volumes will become unused
5839                 if ($virtdev_hash->{$ds}) {
5840                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5841                     if (my $err = $@) {
5842                         warn $err;
5843                     }
5844                 }
5845             });
5846
5847             # delete vmstate files
5848             # since after the restore we have no snapshots anymore
5849             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5850                 my $snap = $oldconf->{snapshots}->{$snapname};
5851                 if ($snap->{vmstate}) {
5852                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5853                     if (my $err = $@) {
5854                         warn $err;
5855                     }
5856                 }
5857             }
5858         }
5859
5860         my $map = {};
5861         foreach my $virtdev (sort keys %$virtdev_hash) {
5862             my $d = $virtdev_hash->{$virtdev};
5863             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5864             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5865
5866             # test if requested format is supported
5867             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5868             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5869             $d->{format} = $defFormat if !$supported;
5870
5871             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5872                                                   $d->{format}, undef, $alloc_size);
5873             print STDERR "new volume ID is '$volid'\n";
5874             $d->{volid} = $volid;
5875             my $path = PVE::Storage::path($cfg, $volid);
5876
5877             PVE::Storage::activate_volumes($cfg,[$volid]);
5878
5879             my $write_zeros = 1;
5880             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5881                 $write_zeros = 0;
5882             }
5883
5884             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5885
5886             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5887             $map->{$virtdev} = $volid;
5888         }
5889
5890         $fh->seek(0, 0) || die "seek failed - $!\n";
5891
5892         my $outfd = new IO::File ($tmpfn, "w") ||
5893             die "unable to write config for VM $vmid\n";
5894
5895         my $cookie = { netcount => 0 };
5896         while (defined(my $line = <$fh>)) {
5897             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5898         }
5899
5900         $fh->close();
5901         $outfd->close();
5902     };
5903
5904     eval {
5905         # enable interrupts
5906         local $SIG{INT} =
5907             local $SIG{TERM} =
5908             local $SIG{QUIT} =
5909             local $SIG{HUP} =
5910             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5911         local $SIG{ALRM} = sub { die "got timeout\n"; };
5912
5913         $oldtimeout = alarm($timeout);
5914
5915         my $parser = sub {
5916             my $line = shift;
5917
5918             print "$line\n";
5919
5920             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5921                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5922                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5923             } elsif ($line =~ m/^CTIME: /) {
5924                 # we correctly received the vma config, so we can disable
5925                 # the timeout now for disk allocation (set to 10 minutes, so
5926                 # that we always timeout if something goes wrong)
5927                 alarm(600);
5928                 &$print_devmap();
5929                 print $fifofh "done\n";
5930                 my $tmp = $oldtimeout || 0;
5931                 $oldtimeout = undef;
5932                 alarm($tmp);
5933                 close($fifofh);
5934             }
5935         };
5936
5937         print "restore vma archive: $cmd\n";
5938         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5939     };
5940     my $err = $@;
5941
5942     alarm($oldtimeout) if $oldtimeout;
5943
5944     my $vollist = [];
5945     foreach my $devname (keys %$devinfo) {
5946         my $volid = $devinfo->{$devname}->{volid};
5947         push @$vollist, $volid if $volid;
5948     }
5949
5950     my $cfg = PVE::Storage::config();
5951     PVE::Storage::deactivate_volumes($cfg, $vollist);
5952
5953     unlink $mapfifo;
5954
5955     if ($err) {
5956         rmtree $tmpdir;
5957         unlink $tmpfn;
5958
5959         foreach my $devname (keys %$devinfo) {
5960             my $volid = $devinfo->{$devname}->{volid};
5961             next if !$volid;
5962             eval {
5963                 if ($volid =~ m|^/|) {
5964                     unlink $volid || die 'unlink failed\n';
5965                 } else {
5966                     PVE::Storage::vdisk_free($cfg, $volid);
5967                 }
5968                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5969             };
5970             print STDERR "unable to cleanup '$volid' - $@" if $@;
5971         }
5972         die $err;
5973     }
5974
5975     rmtree $tmpdir;
5976
5977     rename($tmpfn, $conffile) ||
5978         die "unable to commit configuration file '$conffile'\n";
5979
5980     PVE::Cluster::cfs_update(); # make sure we read new file
5981
5982     eval { rescan($vmid, 1); };
5983     warn $@ if $@;
5984 }
5985
5986 sub restore_tar_archive {
5987     my ($archive, $vmid, $user, $opts) = @_;
5988
5989     if ($archive ne '-') {
5990         my $firstfile = tar_archive_read_firstfile($archive);
5991         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5992             if $firstfile ne 'qemu-server.conf';
5993     }
5994
5995     my $storecfg = PVE::Storage::config();
5996
5997     # destroy existing data - keep empty config
5998     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5999     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6000
6001     my $tocmd = "/usr/lib/qemu-server/qmextract";
6002
6003     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6004     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6005     $tocmd .= ' --prealloc' if $opts->{prealloc};
6006     $tocmd .= ' --info' if $opts->{info};
6007
6008     # tar option "xf" does not autodetect compression when read from STDIN,
6009     # so we pipe to zcat
6010     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6011         PVE::Tools::shellquote("--to-command=$tocmd");
6012
6013     my $tmpdir = "/var/tmp/vzdumptmp$$";
6014     mkpath $tmpdir;
6015
6016     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6017     local $ENV{VZDUMP_VMID} = $vmid;
6018     local $ENV{VZDUMP_USER} = $user;
6019
6020     my $conffile = PVE::QemuConfig->config_file($vmid);
6021     my $tmpfn = "$conffile.$$.tmp";
6022
6023     # disable interrupts (always do cleanups)
6024     local $SIG{INT} =
6025         local $SIG{TERM} =
6026         local $SIG{QUIT} =
6027         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6028
6029     eval {
6030         # enable interrupts
6031         local $SIG{INT} =
6032             local $SIG{TERM} =
6033             local $SIG{QUIT} =
6034             local $SIG{HUP} =
6035             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6036
6037         if ($archive eq '-') {
6038             print "extracting archive from STDIN\n";
6039             run_command($cmd, input => "<&STDIN");
6040         } else {
6041             print "extracting archive '$archive'\n";
6042             run_command($cmd);
6043         }
6044
6045         return if $opts->{info};
6046
6047         # read new mapping
6048         my $map = {};
6049         my $statfile = "$tmpdir/qmrestore.stat";
6050         if (my $fd = IO::File->new($statfile, "r")) {
6051             while (defined (my $line = <$fd>)) {
6052                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6053                     $map->{$1} = $2 if $1;
6054                 } else {
6055                     print STDERR "unable to parse line in statfile - $line\n";
6056                 }
6057             }
6058             $fd->close();
6059         }
6060
6061         my $confsrc = "$tmpdir/qemu-server.conf";
6062
6063         my $srcfd = new IO::File($confsrc, "r") ||
6064             die "unable to open file '$confsrc'\n";
6065
6066         my $outfd = new IO::File ($tmpfn, "w") ||
6067             die "unable to write config for VM $vmid\n";
6068
6069         my $cookie = { netcount => 0 };
6070         while (defined (my $line = <$srcfd>)) {
6071             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6072         }
6073
6074         $srcfd->close();
6075         $outfd->close();
6076     };
6077     my $err = $@;
6078
6079     if ($err) {
6080
6081         unlink $tmpfn;
6082
6083         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6084
6085         die $err;
6086     }
6087
6088     rmtree $tmpdir;
6089
6090     rename $tmpfn, $conffile ||
6091         die "unable to commit configuration file '$conffile'\n";
6092
6093     PVE::Cluster::cfs_update(); # make sure we read new file
6094
6095     eval { rescan($vmid, 1); };
6096     warn $@ if $@;
6097 };
6098
6099 sub foreach_storage_used_by_vm {
6100     my ($conf, $func) = @_;
6101
6102     my $sidhash = {};
6103
6104     foreach_drive($conf, sub {
6105         my ($ds, $drive) = @_;
6106         return if drive_is_cdrom($drive);
6107
6108         my $volid = $drive->{file};
6109
6110         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6111         $sidhash->{$sid} = $sid if $sid;
6112     });
6113
6114     foreach my $sid (sort keys %$sidhash) {
6115         &$func($sid);
6116     }
6117 }
6118
6119 sub do_snapshots_with_qemu {
6120     my ($storecfg, $volid) = @_;
6121
6122     my $storage_name = PVE::Storage::parse_volume_id($volid);
6123
6124     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6125         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6126         return 1;
6127     }
6128
6129     if ($volid =~ m/\.(qcow2|qed)$/){
6130         return 1;
6131     }
6132
6133     return undef;
6134 }
6135
6136 sub qga_check_running {
6137     my ($vmid) = @_;
6138
6139     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6140     if ($@) {
6141         warn "Qemu Guest Agent is not running - $@";
6142         return 0;
6143     }
6144     return 1;
6145 }
6146
6147 sub template_create {
6148     my ($vmid, $conf, $disk) = @_;
6149
6150     my $storecfg = PVE::Storage::config();
6151
6152     foreach_drive($conf, sub {
6153         my ($ds, $drive) = @_;
6154
6155         return if drive_is_cdrom($drive);
6156         return if $disk && $ds ne $disk;
6157
6158         my $volid = $drive->{file};
6159         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6160
6161         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6162         $drive->{file} = $voliddst;
6163         $conf->{$ds} = print_drive($vmid, $drive);
6164         PVE::QemuConfig->write_config($vmid, $conf);
6165     });
6166 }
6167
6168 sub qemu_img_convert {
6169     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6170
6171     my $storecfg = PVE::Storage::config();
6172     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6173     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6174
6175     if ($src_storeid && $dst_storeid) {
6176
6177         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6178
6179         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6180         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6181
6182         my $src_format = qemu_img_format($src_scfg, $src_volname);
6183         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6184
6185         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6186         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6187
6188         my $cmd = [];
6189         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6190         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6191         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6192         if ($is_zero_initialized) {
6193             push @$cmd, "zeroinit:$dst_path";
6194         } else {
6195             push @$cmd, $dst_path;
6196         }
6197
6198         my $parser = sub {
6199             my $line = shift;
6200             if($line =~ m/\((\S+)\/100\%\)/){
6201                 my $percent = $1;
6202                 my $transferred = int($size * $percent / 100);
6203                 my $remaining = $size - $transferred;
6204
6205                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6206             }
6207
6208         };
6209
6210         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6211         my $err = $@;
6212         die "copy failed: $err" if $err;
6213     }
6214 }
6215
6216 sub qemu_img_format {
6217     my ($scfg, $volname) = @_;
6218
6219     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6220         return $1;
6221     } else {
6222         return "raw";
6223     }
6224 }
6225
6226 sub qemu_drive_mirror {
6227     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6228
6229     $jobs = {} if !$jobs;
6230
6231     my $qemu_target;
6232     my $format;
6233     $jobs->{"drive-$drive"} = {};
6234
6235     if ($dst_volid =~ /^nbd:/) {
6236         $qemu_target = $dst_volid;
6237         $format = "nbd";
6238     } else {
6239         my $storecfg = PVE::Storage::config();
6240         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6241
6242         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6243
6244         $format = qemu_img_format($dst_scfg, $dst_volname);
6245
6246         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6247
6248         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6249     }
6250
6251     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6252     $opts->{format} = $format if $format;
6253
6254     print "drive mirror is starting for drive-$drive\n";
6255
6256     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6257
6258     if (my $err = $@) {
6259         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6260         die "mirroring error: $err";
6261     }
6262
6263     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6264 }
6265
6266 sub qemu_drive_mirror_monitor {
6267     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6268
6269     eval {
6270         my $err_complete = 0;
6271
6272         while (1) {
6273             die "storage migration timed out\n" if $err_complete > 300;
6274
6275             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6276
6277             my $running_mirror_jobs = {};
6278             foreach my $stat (@$stats) {
6279                 next if $stat->{type} ne 'mirror';
6280                 $running_mirror_jobs->{$stat->{device}} = $stat;
6281             }
6282
6283             my $readycounter = 0;
6284
6285             foreach my $job (keys %$jobs) {
6286
6287                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6288                     print "$job : finished\n";
6289                     delete $jobs->{$job};
6290                     next;
6291                 }
6292
6293                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6294
6295                 my $busy = $running_mirror_jobs->{$job}->{busy};
6296                 my $ready = $running_mirror_jobs->{$job}->{ready};
6297                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6298                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6299                     my $remaining = $total - $transferred;
6300                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6301
6302                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6303                 }
6304
6305                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6306             }
6307
6308             last if scalar(keys %$jobs) == 0;
6309
6310             if ($readycounter == scalar(keys %$jobs)) {
6311                 print "all mirroring jobs are ready \n";
6312                 last if $skipcomplete; #do the complete later
6313
6314                 if ($vmiddst && $vmiddst != $vmid) {
6315                     my $agent_running = $qga && qga_check_running($vmid);
6316                     if ($agent_running) {
6317                         print "freeze filesystem\n";
6318                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6319                     } else {
6320                         print "suspend vm\n";
6321                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6322                     }
6323
6324                     # if we clone a disk for a new target vm, we don't switch the disk
6325                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6326
6327                     if ($agent_running) {
6328                         print "unfreeze filesystem\n";
6329                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6330                     } else {
6331                         print "resume vm\n";
6332                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6333                     }
6334
6335                     last;
6336                 } else {
6337
6338                     foreach my $job (keys %$jobs) {
6339                         # try to switch the disk if source and destination are on the same guest
6340                         print "$job: Completing block job...\n";
6341
6342                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6343                         if ($@ =~ m/cannot be completed/) {
6344                             print "$job: Block job cannot be completed, try again.\n";
6345                             $err_complete++;
6346                         }else {
6347                             print "$job: Completed successfully.\n";
6348                             $jobs->{$job}->{complete} = 1;
6349                         }
6350                     }
6351                 }
6352             }
6353             sleep 1;
6354         }
6355     };
6356     my $err = $@;
6357
6358     if ($err) {
6359         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6360         die "mirroring error: $err";
6361     }
6362
6363 }
6364
6365 sub qemu_blockjobs_cancel {
6366     my ($vmid, $jobs) = @_;
6367
6368     foreach my $job (keys %$jobs) {
6369         print "$job: Cancelling block job\n";
6370         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6371         $jobs->{$job}->{cancel} = 1;
6372     }
6373
6374     while (1) {
6375         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6376
6377         my $running_jobs = {};
6378         foreach my $stat (@$stats) {
6379             $running_jobs->{$stat->{device}} = $stat;
6380         }
6381
6382         foreach my $job (keys %$jobs) {
6383
6384             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6385                 print "$job: Done.\n";
6386                 delete $jobs->{$job};
6387             }
6388         }
6389
6390         last if scalar(keys %$jobs) == 0;
6391
6392         sleep 1;
6393     }
6394 }
6395
6396 sub clone_disk {
6397     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6398         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6399
6400     my $newvolid;
6401
6402     if (!$full) {
6403         print "create linked clone of drive $drivename ($drive->{file})\n";
6404         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6405         push @$newvollist, $newvolid;
6406     } else {
6407
6408         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6409         $storeid = $storage if $storage;
6410
6411         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6412         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6413
6414         print "create full clone of drive $drivename ($drive->{file})\n";
6415         my $name = undef;
6416         if (drive_is_cloudinit($drive)) {
6417             $name = "vm-$newvmid-cloudinit";
6418             # cloudinit only supports raw and qcow2 atm:
6419             if ($dst_format eq 'qcow2') {
6420                 $name .= '.qcow2';
6421             } elsif ($dst_format ne 'raw') {
6422                 die "clone: unhandled format for cloudinit image\n";
6423             }
6424         }
6425         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6426         push @$newvollist, $newvolid;
6427
6428         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6429
6430         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6431         if (!$running || $snapname) {
6432             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6433         } else {
6434
6435             my $kvmver = get_running_qemu_version ($vmid);
6436             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6437                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6438                     if $drive->{iothread};
6439             }
6440
6441             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6442         }
6443     }
6444
6445     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6446
6447     my $disk = $drive;
6448     $disk->{format} = undef;
6449     $disk->{file} = $newvolid;
6450     $disk->{size} = $size;
6451
6452     return $disk;
6453 }
6454
6455 # this only works if VM is running
6456 sub get_current_qemu_machine {
6457     my ($vmid) = @_;
6458
6459     my $cmd = { execute => 'query-machines', arguments => {} };
6460     my $res = vm_qmp_command($vmid, $cmd);
6461
6462     my ($current, $default);
6463     foreach my $e (@$res) {
6464         $default = $e->{name} if $e->{'is-default'};
6465         $current = $e->{name} if $e->{'is-current'};
6466     }
6467
6468     # fallback to the default machine if current is not supported by qemu
6469     return $current || $default || 'pc';
6470 }
6471
6472 sub get_running_qemu_version {
6473     my ($vmid) = @_;
6474     my $cmd = { execute => 'query-version', arguments => {} };
6475     my $res = vm_qmp_command($vmid, $cmd);
6476     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6477 }
6478
6479 sub qemu_machine_feature_enabled {
6480     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6481
6482     my $current_major;
6483     my $current_minor;
6484
6485     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6486
6487         $current_major = $3;
6488         $current_minor = $4;
6489
6490     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6491
6492         $current_major = $1;
6493         $current_minor = $2;
6494     }
6495
6496     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6497
6498
6499 }
6500
6501 sub qemu_machine_pxe {
6502     my ($vmid, $conf, $machine) = @_;
6503
6504     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6505
6506     foreach my $opt (keys %$conf) {
6507         next if $opt !~ m/^net(\d+)$/;
6508         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6509         next if !$net;
6510         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6511         return $machine.".pxe" if $romfile =~ m/pxe/;
6512         last;
6513     }
6514
6515     return $machine;
6516 }
6517
6518 sub qemu_use_old_bios_files {
6519     my ($machine_type) = @_;
6520
6521     return if !$machine_type;
6522
6523     my $use_old_bios_files = undef;
6524
6525     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6526         $machine_type = $1;
6527         $use_old_bios_files = 1;
6528     } else {
6529         my $kvmver = kvm_user_version();
6530         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6531         # load new efi bios files on migration. So this hack is required to allow
6532         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6533         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6534         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6535     }
6536
6537     return ($use_old_bios_files, $machine_type);
6538 }
6539
6540 sub create_efidisk {
6541     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6542
6543     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6544
6545     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6546     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6547     PVE::Storage::activate_volumes($storecfg, [$volid]);
6548
6549     my $path = PVE::Storage::path($storecfg, $volid);
6550     eval {
6551         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6552     };
6553     die "Copying EFI vars image failed: $@" if $@;
6554
6555     return ($volid, $vars_size);
6556 }
6557
6558 sub lspci {
6559
6560     my $devices = {};
6561
6562     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6563             my (undef, $id, $function) = @_;
6564             my $res = { id => $id, function => $function};
6565             push @{$devices->{$id}}, $res;
6566     });
6567
6568     # Entries should be sorted by functions.
6569     foreach my $id (keys %$devices) {
6570         my $dev = $devices->{$id};
6571         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6572     }
6573
6574     return $devices;
6575 }
6576
6577 sub vm_iothreads_list {
6578     my ($vmid) = @_;
6579
6580     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6581
6582     my $iothreads = {};
6583     foreach my $iothread (@$res) {
6584         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6585     }
6586
6587     return $iothreads;
6588 }
6589
6590 sub scsihw_infos {
6591     my ($conf, $drive) = @_;
6592
6593     my $maxdev = 0;
6594
6595     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6596         $maxdev = 7;
6597     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6598         $maxdev = 1;
6599     } else {
6600         $maxdev = 256;
6601     }
6602
6603     my $controller = int($drive->{index} / $maxdev);
6604     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6605
6606     return ($maxdev, $controller, $controller_prefix);
6607 }
6608
6609 sub add_hyperv_enlightenments {
6610     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6611
6612     return if $winversion < 6;
6613     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6614
6615     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6616
6617     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6618         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6619         push @$cpuFlags , 'hv_vapic';
6620         push @$cpuFlags , 'hv_time';
6621     } else {
6622         push @$cpuFlags , 'hv_spinlocks=0xffff';
6623     }
6624
6625     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6626         push @$cpuFlags , 'hv_reset';
6627         push @$cpuFlags , 'hv_vpindex';
6628         push @$cpuFlags , 'hv_runtime';
6629     }
6630
6631     if ($winversion >= 7) {
6632         push @$cpuFlags , 'hv_relaxed';
6633     }
6634 }
6635
6636 sub windows_version {
6637     my ($ostype) = @_;
6638
6639     return 0 if !$ostype;
6640
6641     my $winversion = 0;
6642
6643     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6644         $winversion = 5;
6645     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6646         $winversion = 6;
6647     } elsif ($ostype =~ m/^win(\d+)$/) {
6648         $winversion = $1;
6649     }
6650
6651     return $winversion;
6652 }
6653
6654 sub resolve_dst_disk_format {
6655         my ($storecfg, $storeid, $src_volname, $format) = @_;
6656         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6657
6658         if (!$format) {
6659             # if no target format is specified, use the source disk format as hint
6660             if ($src_volname) {
6661                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6662                 $format = qemu_img_format($scfg, $src_volname);
6663             } else {
6664                 return $defFormat;
6665             }
6666         }
6667
6668         # test if requested format is supported - else use default
6669         my $supported = grep { $_ eq $format } @$validFormats;
6670         $format = $defFormat if !$supported;
6671         return $format;
6672 }
6673
6674 sub resolve_first_disk {
6675     my $conf = shift;
6676     my @disks = PVE::QemuServer::valid_drive_names();
6677     my $firstdisk;
6678     foreach my $ds (reverse @disks) {
6679         next if !$conf->{$ds};
6680         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6681         next if PVE::QemuServer::drive_is_cdrom($disk);
6682         $firstdisk = $ds;
6683     }
6684     return $firstdisk;
6685 }
6686
6687 sub generate_smbios1_uuid {
6688     my ($uuid, $uuid_str);
6689     UUID::generate($uuid);
6690     UUID::unparse($uuid, $uuid_str);
6691     return "uuid=$uuid_str";
6692 }
6693
6694 # bash completion helper
6695
6696 sub complete_backup_archives {
6697     my ($cmdname, $pname, $cvalue) = @_;
6698
6699     my $cfg = PVE::Storage::config();
6700
6701     my $storeid;
6702
6703     if ($cvalue =~ m/^([^:]+):/) {
6704         $storeid = $1;
6705     }
6706
6707     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6708
6709     my $res = [];
6710     foreach my $id (keys %$data) {
6711         foreach my $item (@{$data->{$id}}) {
6712             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6713             push @$res, $item->{volid} if defined($item->{volid});
6714         }
6715     }
6716
6717     return $res;
6718 }
6719
6720 my $complete_vmid_full = sub {
6721     my ($running) = @_;
6722
6723     my $idlist = vmstatus();
6724
6725     my $res = [];
6726
6727     foreach my $id (keys %$idlist) {
6728         my $d = $idlist->{$id};
6729         if (defined($running)) {
6730             next if $d->{template};
6731             next if $running && $d->{status} ne 'running';
6732             next if !$running && $d->{status} eq 'running';
6733         }
6734         push @$res, $id;
6735
6736     }
6737     return $res;
6738 };
6739
6740 sub complete_vmid {
6741     return &$complete_vmid_full();
6742 }
6743
6744 sub complete_vmid_stopped {
6745     return &$complete_vmid_full(0);
6746 }
6747
6748 sub complete_vmid_running {
6749     return &$complete_vmid_full(1);
6750 }
6751
6752 sub complete_storage {
6753
6754     my $cfg = PVE::Storage::config();
6755     my $ids = $cfg->{ids};
6756
6757     my $res = [];
6758     foreach my $sid (keys %$ids) {
6759         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6760         next if !$ids->{$sid}->{content}->{images};
6761         push @$res, $sid;
6762     }
6763
6764     return $res;
6765 }
6766
6767 1;