PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  42 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  43 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  44
  45 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  46
  47 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  48
  49 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  50
  51 # Note about locking: we use flock on the config file protect
  52 # against concurent actions.
  53 # Aditionaly, we have a 'lock' setting in the config file. This
  54 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  55 # allowed when such lock is set. But you can ignore this kind of
  56 # lock with the --skiplock flag.
  57
  58 cfs_register_file('/qemu-server/',
  59                   \&parse_vm_config,
  60                   \&write_vm_config);
  61
  62 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  63     description => "Some command save/restore state from this location.",
  64     type => 'string',
  65     maxLength => 128,
  66     optional => 1,
  67 });
  68
  69 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  70     description => "The name of the snapshot.",
  71     type => 'string', format => 'pve-configid',
  72     maxLength => 40,
  73 });
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  76     type => 'string',
  77     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  78     description => "The drive's backing file's data format.",
  79     optional => 1,
  80 });
  81
  82 #no warnings 'redefine';
  83
  84 sub cgroups_write {
  85    my ($controller, $vmid, $option, $value) = @_;
  86
  87    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  88    PVE::ProcFSTools::write_proc_entry($path, $value);
  89
  90 }
  91
  92 my $nodename = PVE::INotify::nodename();
  93
  94 mkdir "/etc/pve/nodes/$nodename";
  95 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  96 mkdir $confdir;
  97
  98 my $var_run_tmpdir = "/var/run/qemu-server";
  99 mkdir $var_run_tmpdir;
 100
 101 my $lock_dir = "/var/lock/qemu-server";
 102 mkdir $lock_dir;
 103
 104 my $pcisysfs = "/sys/bus/pci";
 105
 106 my $cpu_vendor_list = {
 107     # Intel CPUs
 108     486 => 'GenuineIntel',
 109     pentium => 'GenuineIntel',
 110     pentium2  => 'GenuineIntel',
 111     pentium3  => 'GenuineIntel',
 112     coreduo => 'GenuineIntel',
 113     core2duo => 'GenuineIntel',
 114     Conroe  => 'GenuineIntel',
 115     Penryn  => 'GenuineIntel',
 116     Nehalem  => 'GenuineIntel',
 117     'Nehalem-IBRS'  => 'GenuineIntel',
 118     Westmere => 'GenuineIntel',
 119     'Westmere-IBRS' => 'GenuineIntel',
 120     SandyBridge => 'GenuineIntel',
 121     'SandyBridge-IBRS' => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     'IvyBridge-IBRS' => 'GenuineIntel',
 124     Haswell => 'GenuineIntel',
 125     'Haswell-IBRS' => 'GenuineIntel',
 126     'Haswell-noTSX' => 'GenuineIntel',
 127     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 128     Broadwell => 'GenuineIntel',
 129     'Broadwell-IBRS' => 'GenuineIntel',
 130     'Broadwell-noTSX' => 'GenuineIntel',
 131     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 132     'Skylake-Client' => 'GenuineIntel',
 133     'Skylake-Client-IBRS' => 'GenuineIntel',
 134     'Skylake-Server' => 'GenuineIntel',
 135     'Skylake-Server-IBRS' => 'GenuineIntel',
 136
 137     # AMD CPUs
 138     athlon => 'AuthenticAMD',
 139     phenom  => 'AuthenticAMD',
 140     Opteron_G1  => 'AuthenticAMD',
 141     Opteron_G2  => 'AuthenticAMD',
 142     Opteron_G3  => 'AuthenticAMD',
 143     Opteron_G4  => 'AuthenticAMD',
 144     Opteron_G5  => 'AuthenticAMD',
 145     EPYC => 'AuthenticAMD',
 146     'EPYC-IBPB' => 'AuthenticAMD',
 147
 148     # generic types, use vendor from host node
 149     host => 'default',
 150     kvm32 => 'default',
 151     kvm64 => 'default',
 152     qemu32 => 'default',
 153     qemu64 => 'default',
 154     max => 'default',
 155 };
 156
 157 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 158
 159 my $cpu_fmt = {
 160     cputype => {
 161         description => "Emulated CPU type.",
 162         type => 'string',
 163         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 164         default => 'kvm64',
 165         default_key => 1,
 166     },
 167     hidden => {
 168         description => "Do not identify as a KVM virtual machine.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 0
 172     },
 173     flags => {
 174         description => "List of additional CPU flags separated by ';'."
 175                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 176                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 177         format_description => '+FLAG[;-FLAG...]',
 178         type => 'string',
 179         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 180         optional => 1,
 181     },
 182 };
 183
 184 my $watchdog_fmt = {
 185     model => {
 186         default_key => 1,
 187         type => 'string',
 188         enum => [qw(i6300esb ib700)],
 189         description => "Watchdog type to emulate.",
 190         default => 'i6300esb',
 191         optional => 1,
 192     },
 193     action => {
 194         type => 'string',
 195         enum => [qw(reset shutdown poweroff pause debug none)],
 196         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 197         optional => 1,
 198     },
 199 };
 200 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 201
 202 my $confdesc = {
 203     onboot => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Specifies whether a VM will be started during system bootup.",
 207         default => 0,
 208     },
 209     autostart => {
 210         optional => 1,
 211         type => 'boolean',
 212         description => "Automatic restart after crash (currently ignored).",
 213         default => 0,
 214     },
 215     hotplug => {
 216         optional => 1,
 217         type => 'string', format => 'pve-hotplug-features',
 218         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 219         default => 'network,disk,usb',
 220     },
 221     reboot => {
 222         optional => 1,
 223         type => 'boolean',
 224         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 225         default => 1,
 226     },
 227     lock => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Lock/unlock the VM.",
 231         enum => [qw(migrate backup snapshot rollback)],
 232     },
 233     cpulimit => {
 234         optional => 1,
 235         type => 'number',
 236         description => "Limit of CPU usage.",
 237         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 238         minimum => 0,
 239         maximum => 128,
 240         default => 0,
 241     },
 242     cpuunits => {
 243         optional => 1,
 244         type => 'integer',
 245         description => "CPU weight for a VM.",
 246         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 247         minimum => 2,
 248         maximum => 262144,
 249         default => 1024,
 250     },
 251     memory => {
 252         optional => 1,
 253         type => 'integer',
 254         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 255         minimum => 16,
 256         default => 512,
 257     },
 258     balloon => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 262         minimum => 0,
 263     },
 264     shares => {
 265         optional => 1,
 266         type => 'integer',
 267         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 268         minimum => 0,
 269         maximum => 50000,
 270         default => 1000,
 271     },
 272     keyboard => {
 273         optional => 1,
 274         type => 'string',
 275         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 276                        "It should not be necessary to set it.",
 277         enum => PVE::Tools::kvmkeymaplist(),
 278         default => undef,
 279     },
 280     name => {
 281         optional => 1,
 282         type => 'string', format => 'dns-name',
 283         description => "Set a name for the VM. Only used on the configuration web interface.",
 284     },
 285     scsihw => {
 286         optional => 1,
 287         type => 'string',
 288         description => "SCSI controller model",
 289         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 290         default => 'lsi',
 291     },
 292     description => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 296     },
 297     ostype => {
 298         optional => 1,
 299         type => 'string',
 300         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 301         description => "Specify guest operating system.",
 302         verbose_description => <<EODESC,
 303 Specify guest operating system. This is used to enable special
 304 optimization/features for specific operating systems:
 305
 306 [horizontal]
 307 other;; unspecified OS
 308 wxp;; Microsoft Windows XP
 309 w2k;; Microsoft Windows 2000
 310 w2k3;; Microsoft Windows 2003
 311 w2k8;; Microsoft Windows 2008
 312 wvista;; Microsoft Windows Vista
 313 win7;; Microsoft Windows 7
 314 win8;; Microsoft Windows 8/2012/2012r2
 315 win10;; Microsoft Windows 10/2016
 316 l24;; Linux 2.4 Kernel
 317 l26;; Linux 2.6/3.X Kernel
 318 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 319 EODESC
 320     },
 321     boot => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 325         pattern => '[acdn]{1,4}',
 326         default => 'cdn',
 327     },
 328     bootdisk => {
 329         optional => 1,
 330         type => 'string', format => 'pve-qm-bootdisk',
 331         description => "Enable booting from specified disk.",
 332         pattern => '(ide|sata|scsi|virtio)\d+',
 333     },
 334     smp => {
 335         optional => 1,
 336         type => 'integer',
 337         description => "The number of CPUs. Please use option -sockets instead.",
 338         minimum => 1,
 339         default => 1,
 340     },
 341     sockets => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "The number of CPU sockets.",
 345         minimum => 1,
 346         default => 1,
 347     },
 348     cores => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "The number of cores per socket.",
 352         minimum => 1,
 353         default => 1,
 354     },
 355     numa => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable NUMA.",
 359         default => 0,
 360     },
 361     hugepages => {
 362         optional => 1,
 363         type => 'string',
 364         description => "Enable/disable hugepages memory.",
 365         enum => [qw(any 2 1024)],
 366     },
 367     vcpus => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Number of hotplugged vcpus.",
 371         minimum => 1,
 372         default => 0,
 373     },
 374     acpi => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable ACPI.",
 378         default => 1,
 379     },
 380     agent => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable Qemu GuestAgent.",
 384         default => 0,
 385     },
 386     kvm => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable KVM hardware virtualization.",
 390         default => 1,
 391     },
 392     tdf => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Enable/disable time drift fix.",
 396         default => 0,
 397     },
 398     localtime => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 402     },
 403     freeze => {
 404         optional => 1,
 405         type => 'boolean',
 406         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 407     },
 408     vga => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Select the VGA type.",
 412         verbose_description => "Select the VGA type. If you want to use high resolution" .
 413             " modes (>= 1280x1024x16) then you should use the options " .
 414             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 415             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 416             "display sever. For win* OS you can select how many independent " .
 417             "displays you want, Linux guests can add displays them self. " .
 418             "You can also run without any graphic card, using a serial device" .
 419             " as terminal.",
 420         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 421     },
 422     watchdog => {
 423         optional => 1,
 424         type => 'string', format => 'pve-qm-watchdog',
 425         description => "Create a virtual hardware watchdog device.",
 426         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 427             " (by a guest action), the watchdog must be periodically polled " .
 428             "by an agent inside the guest or else the watchdog will reset " .
 429             "the guest (or execute the respective action specified)",
 430     },
 431     startdate => {
 432         optional => 1,
 433         type => 'string',
 434         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 435         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 436         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 437         default => 'now',
 438     },
 439     startup =>  get_standard_option('pve-startup-order'),
 440     template => {
 441         optional => 1,
 442         type => 'boolean',
 443         description => "Enable/disable Template.",
 444         default => 0,
 445     },
 446     args => {
 447         optional => 1,
 448         type => 'string',
 449         description => "Arbitrary arguments passed to kvm.",
 450         verbose_description => <<EODESCR,
 451 Arbitrary arguments passed to kvm, for example:
 452
 453 args: -no-reboot -no-hpet
 454
 455 NOTE: this option is for experts only.
 456 EODESCR
 457     },
 458     tablet => {
 459         optional => 1,
 460         type => 'boolean',
 461         default => 1,
 462         description => "Enable/disable the USB tablet device.",
 463         verbose_description => "Enable/disable the USB tablet device. This device is " .
 464             "usually needed to allow absolute mouse positioning with VNC. " .
 465             "Else the mouse runs out of sync with normal VNC clients. " .
 466             "If you're running lots of console-only guests on one host, " .
 467             "you may consider disabling this to save some context switches. " .
 468             "This is turned off by default if you use spice (-vga=qxl).",
 469     },
 470     migrate_speed => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 474         minimum => 0,
 475         default => 0,
 476     },
 477     migrate_downtime => {
 478         optional => 1,
 479         type => 'number',
 480         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 481         minimum => 0,
 482         default => 0.1,
 483     },
 484     cdrom => {
 485         optional => 1,
 486         type => 'string', format => 'pve-qm-ide',
 487         typetext => '<volume>',
 488         description => "This is an alias for option -ide2",
 489     },
 490     cpu => {
 491         optional => 1,
 492         description => "Emulated CPU type.",
 493         type => 'string',
 494         format => $cpu_fmt,
 495     },
 496     parent => get_standard_option('pve-snapshot-name', {
 497         optional => 1,
 498         description => "Parent snapshot name. This is used internally, and should not be modified.",
 499     }),
 500     snaptime => {
 501         optional => 1,
 502         description => "Timestamp for snapshots.",
 503         type => 'integer',
 504         minimum => 0,
 505     },
 506     vmstate => {
 507         optional => 1,
 508         type => 'string', format => 'pve-volume-id',
 509         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 510     },
 511     vmstatestorage => get_standard_option('pve-storage-id', {
 512         description => "Default storage for VM state volumes/files.",
 513         optional => 1,
 514     }),
 515     machine => {
 516         description => "Specific the Qemu machine type.",
 517         type => 'string',
 518         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 519         maxLength => 40,
 520         optional => 1,
 521     },
 522     smbios1 => {
 523         description => "Specify SMBIOS type 1 fields.",
 524         type => 'string', format => 'pve-qm-smbios1',
 525         maxLength => 256,
 526         optional => 1,
 527     },
 528     protection => {
 529         optional => 1,
 530         type => 'boolean',
 531         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 532         default => 0,
 533     },
 534     bios => {
 535         optional => 1,
 536         type => 'string',
 537         enum => [ qw(seabios ovmf) ],
 538         description => "Select BIOS implementation.",
 539         default => 'seabios',
 540     },
 541 };
 542
 543 my $confdesc_cloudinit = {
 544     citype => {
 545         optional => 1,
 546         type => 'string',
 547         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 548         enum => ['configdrive2', 'nocloud'],
 549     },
 550     ciuser => {
 551         optional => 1,
 552         type => 'string',
 553         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 554     },
 555     cipassword => {
 556         optional => 1,
 557         type => 'string',
 558         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 559     },
 560     searchdomain => {
 561         optional => 1,
 562         type => 'string',
 563         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 564     },
 565     nameserver => {
 566         optional => 1,
 567         type => 'string', format => 'address-list',
 568         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 569     },
 570     sshkeys => {
 571         optional => 1,
 572         type => 'string',
 573         format => 'urlencoded',
 574         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 575     },
 576 };
 577
 578 # what about other qemu settings ?
 579 #cpu => 'string',
 580 #machine => 'string',
 581 #fda => 'file',
 582 #fdb => 'file',
 583 #mtdblock => 'file',
 584 #sd => 'file',
 585 #pflash => 'file',
 586 #snapshot => 'bool',
 587 #bootp => 'file',
 588 ##tftp => 'dir',
 589 ##smb => 'dir',
 590 #kernel => 'file',
 591 #append => 'string',
 592 #initrd => 'file',
 593 ##soundhw => 'string',
 594
 595 while (my ($k, $v) = each %$confdesc) {
 596     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 597 }
 598
 599 my $MAX_IDE_DISKS = 4;
 600 my $MAX_SCSI_DISKS = 14;
 601 my $MAX_VIRTIO_DISKS = 16;
 602 my $MAX_SATA_DISKS = 6;
 603 my $MAX_USB_DEVICES = 5;
 604 my $MAX_NETS = 32;
 605 my $MAX_UNUSED_DISKS = 8;
 606 my $MAX_HOSTPCI_DEVICES = 4;
 607 my $MAX_SERIAL_PORTS = 4;
 608 my $MAX_PARALLEL_PORTS = 3;
 609 my $MAX_NUMA = 8;
 610
 611 my $numa_fmt = {
 612     cpus => {
 613         type => "string",
 614         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 615         description => "CPUs accessing this NUMA node.",
 616         format_description => "id[-id];...",
 617     },
 618     memory => {
 619         type => "number",
 620         description => "Amount of memory this NUMA node provides.",
 621         optional => 1,
 622     },
 623     hostnodes => {
 624         type => "string",
 625         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 626         description => "Host NUMA nodes to use.",
 627         format_description => "id[-id];...",
 628         optional => 1,
 629     },
 630     policy => {
 631         type => 'string',
 632         enum => [qw(preferred bind interleave)],
 633         description => "NUMA allocation policy.",
 634         optional => 1,
 635     },
 636 };
 637 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 638 my $numadesc = {
 639     optional => 1,
 640     type => 'string', format => $numa_fmt,
 641     description => "NUMA topology.",
 642 };
 643 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 644
 645 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 646     $confdesc->{"numa$i"} = $numadesc;
 647 }
 648
 649 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 650                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 651                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 652 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 653
 654 my $net_fmt_bridge_descr = <<__EOD__;
 655 Bridge to attach the network device to. The Proxmox VE standard bridge
 656 is called 'vmbr0'.
 657
 658 If you do not specify a bridge, we create a kvm user (NATed) network
 659 device, which provides DHCP and DNS services. The following addresses
 660 are used:
 661
 662  10.0.2.2   Gateway
 663  10.0.2.3   DNS Server
 664  10.0.2.4   SMB Server
 665
 666 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 667 __EOD__
 668
 669 my $net_fmt = {
 670     macaddr => {
 671         type => 'string',
 672         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 673         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 674         format_description => "XX:XX:XX:XX:XX:XX",
 675         optional => 1,
 676     },
 677     model => {
 678         type => 'string',
 679         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 680         enum => $nic_model_list,
 681         default_key => 1,
 682     },
 683     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 684     bridge => {
 685         type => 'string',
 686         description => $net_fmt_bridge_descr,
 687         format_description => 'bridge',
 688         optional => 1,
 689     },
 690     queues => {
 691         type => 'integer',
 692         minimum => 0, maximum => 16,
 693         description => 'Number of packet queues to be used on the device.',
 694         optional => 1,
 695     },
 696     rate => {
 697         type => 'number',
 698         minimum => 0,
 699         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 700         optional => 1,
 701     },
 702     tag => {
 703         type => 'integer',
 704         minimum => 1, maximum => 4094,
 705         description => 'VLAN tag to apply to packets on this interface.',
 706         optional => 1,
 707     },
 708     trunks => {
 709         type => 'string',
 710         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 711         description => 'VLAN trunks to pass through this interface.',
 712         format_description => 'vlanid[;vlanid...]',
 713         optional => 1,
 714     },
 715     firewall => {
 716         type => 'boolean',
 717         description => 'Whether this interface should be protected by the firewall.',
 718         optional => 1,
 719     },
 720     link_down => {
 721         type => 'boolean',
 722         description => 'Whether this interface should be disconnected (like pulling the plug).',
 723         optional => 1,
 724     },
 725 };
 726
 727 my $netdesc = {
 728     optional => 1,
 729     type => 'string', format => $net_fmt,
 730     description => "Specify network devices.",
 731 };
 732
 733 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 734
 735 my $ipconfig_fmt = {
 736     ip => {
 737         type => 'string',
 738         format => 'pve-ipv4-config',
 739         format_description => 'IPv4Format/CIDR',
 740         description => 'IPv4 address in CIDR format.',
 741         optional => 1,
 742         default => 'dhcp',
 743     },
 744     gw => {
 745         type => 'string',
 746         format => 'ipv4',
 747         format_description => 'GatewayIPv4',
 748         description => 'Default gateway for IPv4 traffic.',
 749         optional => 1,
 750         requires => 'ip',
 751     },
 752     ip6 => {
 753         type => 'string',
 754         format => 'pve-ipv6-config',
 755         format_description => 'IPv6Format/CIDR',
 756         description => 'IPv6 address in CIDR format.',
 757         optional => 1,
 758         default => 'dhcp',
 759     },
 760     gw6 => {
 761         type => 'string',
 762         format => 'ipv6',
 763         format_description => 'GatewayIPv6',
 764         description => 'Default gateway for IPv6 traffic.',
 765         optional => 1,
 766         requires => 'ip6',
 767     },
 768 };
 769 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 770 my $ipconfigdesc = {
 771     optional => 1,
 772     type => 'string', format => 'pve-qm-ipconfig',
 773     description => <<'EODESCR',
 774 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 775
 776 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 777
 778 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 779 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 780
 781 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 782 EODESCR
 783 };
 784 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 785
 786 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 787     $confdesc->{"net$i"} = $netdesc;
 788     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 789 }
 790
 791 foreach my $key (keys %$confdesc_cloudinit) {
 792     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 793 }
 794
 795 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 796 sub verify_volume_id_or_qm_path {
 797     my ($volid, $noerr) = @_;
 798
 799     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 800         return $volid;
 801     }
 802
 803     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 804     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 805     if ($@) {
 806         return undef if $noerr;
 807         die $@;
 808     }
 809     return $volid;
 810 }
 811
 812 my $drivename_hash;
 813
 814 my %drivedesc_base = (
 815     volume => { alias => 'file' },
 816     file => {
 817         type => 'string',
 818         format => 'pve-volume-id-or-qm-path',
 819         default_key => 1,
 820         format_description => 'volume',
 821         description => "The drive's backing volume.",
 822     },
 823     media => {
 824         type => 'string',
 825         enum => [qw(cdrom disk)],
 826         description => "The drive's media type.",
 827         default => 'disk',
 828         optional => 1
 829     },
 830     cyls => {
 831         type => 'integer',
 832         description => "Force the drive's physical geometry to have a specific cylinder count.",
 833         optional => 1
 834     },
 835     heads => {
 836         type => 'integer',
 837         description => "Force the drive's physical geometry to have a specific head count.",
 838         optional => 1
 839     },
 840     secs => {
 841         type => 'integer',
 842         description => "Force the drive's physical geometry to have a specific sector count.",
 843         optional => 1
 844     },
 845     trans => {
 846         type => 'string',
 847         enum => [qw(none lba auto)],
 848         description => "Force disk geometry bios translation mode.",
 849         optional => 1,
 850     },
 851     snapshot => {
 852         type => 'boolean',
 853         description => "Controls qemu's snapshot mode feature."
 854             . " If activated, changes made to the disk are temporary and will"
 855             . " be discarded when the VM is shutdown.",
 856         optional => 1,
 857     },
 858     cache => {
 859         type => 'string',
 860         enum => [qw(none writethrough writeback unsafe directsync)],
 861         description => "The drive's cache mode",
 862         optional => 1,
 863     },
 864     format => get_standard_option('pve-qm-image-format'),
 865     size => {
 866         type => 'string',
 867         format => 'disk-size',
 868         format_description => 'DiskSize',
 869         description => "Disk size. This is purely informational and has no effect.",
 870         optional => 1,
 871     },
 872     backup => {
 873         type => 'boolean',
 874         description => "Whether the drive should be included when making backups.",
 875         optional => 1,
 876     },
 877     replicate => {
 878         type => 'boolean',
 879         description => 'Whether the drive should considered for replication jobs.',
 880         optional => 1,
 881         default => 1,
 882     },
 883     rerror => {
 884         type => 'string',
 885         enum => [qw(ignore report stop)],
 886         description => 'Read error action.',
 887         optional => 1,
 888     },
 889     werror => {
 890         type => 'string',
 891         enum => [qw(enospc ignore report stop)],
 892         description => 'Write error action.',
 893         optional => 1,
 894     },
 895     aio => {
 896         type => 'string',
 897         enum => [qw(native threads)],
 898         description => 'AIO type to use.',
 899         optional => 1,
 900     },
 901     discard => {
 902         type => 'string',
 903         enum => [qw(ignore on)],
 904         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 905         optional => 1,
 906     },
 907     detect_zeroes => {
 908         type => 'boolean',
 909         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 910         optional => 1,
 911     },
 912     serial => {
 913         type => 'string',
 914         format => 'urlencoded',
 915         format_description => 'serial',
 916         maxLength => 20*3, # *3 since it's %xx url enoded
 917         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 918         optional => 1,
 919     },
 920     shared => {
 921         type => 'boolean',
 922         description => 'Mark this locally-managed volume as available on all nodes',
 923         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 924         optional => 1,
 925         default => 0,
 926     }
 927 );
 928
 929 my %iothread_fmt = ( iothread => {
 930         type => 'boolean',
 931         description => "Whether to use iothreads for this drive",
 932         optional => 1,
 933 });
 934
 935 my %model_fmt = (
 936     model => {
 937         type => 'string',
 938         format => 'urlencoded',
 939         format_description => 'model',
 940         maxLength => 40*3, # *3 since it's %xx url enoded
 941         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 942         optional => 1,
 943     },
 944 );
 945
 946 my %queues_fmt = (
 947     queues => {
 948         type => 'integer',
 949         description => "Number of queues.",
 950         minimum => 2,
 951         optional => 1
 952     }
 953 );
 954
 955 my %scsiblock_fmt = (
 956     scsiblock => {
 957         type => 'boolean',
 958         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 959         optional => 1,
 960         default => 0,
 961     },
 962 );
 963
 964 my $add_throttle_desc = sub {
 965     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 966     my $d = {
 967         type => $type,
 968         format_description => $unit,
 969         description => "Maximum $what in $longunit.",
 970         optional => 1,
 971     };
 972     $d->{minimum} = $minimum if defined($minimum);
 973     $drivedesc_base{$key} = $d;
 974 };
 975 # throughput: (leaky bucket)
 976 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 979 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 982 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 983 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 984 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 985
 986 # pools: (pool of IO before throttling starts taking effect)
 987 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 990 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 991 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 992 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 993
 994 # burst lengths
 995 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1001
1002 # legacy support
1003 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1004 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1005 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1006 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1007
1008 my $ide_fmt = {
1009     %drivedesc_base,
1010     %model_fmt,
1011 };
1012 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1013
1014 my $idedesc = {
1015     optional => 1,
1016     type => 'string', format => $ide_fmt,
1017     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1018 };
1019 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1020
1021 my $scsi_fmt = {
1022     %drivedesc_base,
1023     %iothread_fmt,
1024     %queues_fmt,
1025     %scsiblock_fmt,
1026 };
1027 my $scsidesc = {
1028     optional => 1,
1029     type => 'string', format => $scsi_fmt,
1030     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1031 };
1032 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1033
1034 my $sata_fmt = {
1035     %drivedesc_base,
1036 };
1037 my $satadesc = {
1038     optional => 1,
1039     type => 'string', format => $sata_fmt,
1040     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1041 };
1042 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1043
1044 my $virtio_fmt = {
1045     %drivedesc_base,
1046     %iothread_fmt,
1047 };
1048 my $virtiodesc = {
1049     optional => 1,
1050     type => 'string', format => $virtio_fmt,
1051     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1054
1055 my $alldrive_fmt = {
1056     %drivedesc_base,
1057     %iothread_fmt,
1058     %model_fmt,
1059     %queues_fmt,
1060     %scsiblock_fmt,
1061 };
1062
1063 my $efidisk_fmt = {
1064     volume => { alias => 'file' },
1065     file => {
1066         type => 'string',
1067         format => 'pve-volume-id-or-qm-path',
1068         default_key => 1,
1069         format_description => 'volume',
1070         description => "The drive's backing volume.",
1071     },
1072     format => get_standard_option('pve-qm-image-format'),
1073     size => {
1074         type => 'string',
1075         format => 'disk-size',
1076         format_description => 'DiskSize',
1077         description => "Disk size. This is purely informational and has no effect.",
1078         optional => 1,
1079     },
1080 };
1081
1082 my $efidisk_desc = {
1083     optional => 1,
1084     type => 'string', format => $efidisk_fmt,
1085     description => "Configure a Disk for storing EFI vars",
1086 };
1087
1088 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1089
1090 my $usb_fmt = {
1091     host => {
1092         default_key => 1,
1093         type => 'string', format => 'pve-qm-usb-device',
1094         format_description => 'HOSTUSBDEVICE|spice',
1095         description => <<EODESCR,
1096 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1097
1098  'bus-port(.port)*' (decimal numbers) or
1099  'vendor_id:product_id' (hexadeciaml numbers) or
1100  'spice'
1101
1102 You can use the 'lsusb -t' command to list existing usb devices.
1103
1104 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1105
1106 The value 'spice' can be used to add a usb redirection devices for spice.
1107 EODESCR
1108     },
1109     usb3 => {
1110         optional => 1,
1111         type => 'boolean',
1112         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1113         default => 0,
1114     },
1115 };
1116
1117 my $usbdesc = {
1118     optional => 1,
1119     type => 'string', format => $usb_fmt,
1120     description => "Configure an USB device (n is 0 to 4).",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1123
1124 # NOTE: the match-groups of this regex are used in parse_hostpci
1125 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1126 my $hostpci_fmt = {
1127     host => {
1128         default_key => 1,
1129         type => 'string',
1130         pattern => qr/$PCIRE(;$PCIRE)*/,
1131         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1132         description => <<EODESCR,
1133 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1134 of PCI virtual functions of the host. HOSTPCIID syntax is:
1135
1136 'bus:dev.func' (hexadecimal numbers)
1137
1138 You can us the 'lspci' command to list existing PCI devices.
1139 EODESCR
1140     },
1141     rombar => {
1142         type => 'boolean',
1143         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1144         optional => 1,
1145         default => 1,
1146     },
1147     romfile => {
1148         type => 'string',
1149         pattern => '[^,;]+',
1150         format_description => 'string',
1151         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1152         optional => 1,
1153     },
1154     pcie => {
1155         type => 'boolean',
1156         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1157         optional => 1,
1158         default => 0,
1159     },
1160     'x-vga' => {
1161         type => 'boolean',
1162         description =>  "Enable vfio-vga device support.",
1163         optional => 1,
1164         default => 0,
1165     },
1166 };
1167 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1168
1169 my $hostpcidesc = {
1170         optional => 1,
1171         type => 'string', format => 'pve-qm-hostpci',
1172         description => "Map host PCI devices into guest.",
1173         verbose_description =>  <<EODESCR,
1174 Map host PCI devices into guest.
1175
1176 NOTE: This option allows direct access to host hardware. So it is no longer
1177 possible to migrate such machines - use with special care.
1178
1179 CAUTION: Experimental! User reported problems with this option.
1180 EODESCR
1181 };
1182 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1183
1184 my $serialdesc = {
1185         optional => 1,
1186         type => 'string',
1187         pattern => '(/dev/.+|socket)',
1188         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1189         verbose_description =>  <<EODESCR,
1190 Create a serial device inside the VM (n is 0 to 3), and pass through a
1191 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1192 host side (use 'qm terminal' to open a terminal connection).
1193
1194 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1195
1196 CAUTION: Experimental! User reported problems with this option.
1197 EODESCR
1198 };
1199
1200 my $paralleldesc= {
1201         optional => 1,
1202         type => 'string',
1203         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1204         description =>  "Map host parallel devices (n is 0 to 2).",
1205         verbose_description =>  <<EODESCR,
1206 Map host parallel devices (n is 0 to 2).
1207
1208 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1209
1210 CAUTION: Experimental! User reported problems with this option.
1211 EODESCR
1212 };
1213
1214 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1215     $confdesc->{"parallel$i"} = $paralleldesc;
1216 }
1217
1218 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1219     $confdesc->{"serial$i"} = $serialdesc;
1220 }
1221
1222 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1223     $confdesc->{"hostpci$i"} = $hostpcidesc;
1224 }
1225
1226 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1227     $drivename_hash->{"ide$i"} = 1;
1228     $confdesc->{"ide$i"} = $idedesc;
1229 }
1230
1231 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1232     $drivename_hash->{"sata$i"} = 1;
1233     $confdesc->{"sata$i"} = $satadesc;
1234 }
1235
1236 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1237     $drivename_hash->{"scsi$i"} = 1;
1238     $confdesc->{"scsi$i"} = $scsidesc ;
1239 }
1240
1241 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1242     $drivename_hash->{"virtio$i"} = 1;
1243     $confdesc->{"virtio$i"} = $virtiodesc;
1244 }
1245
1246 $drivename_hash->{efidisk0} = 1;
1247 $confdesc->{efidisk0} = $efidisk_desc;
1248
1249 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1250     $confdesc->{"usb$i"} = $usbdesc;
1251 }
1252
1253 my $unuseddesc = {
1254     optional => 1,
1255     type => 'string', format => 'pve-volume-id',
1256     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1257 };
1258
1259 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1260     $confdesc->{"unused$i"} = $unuseddesc;
1261 }
1262
1263 my $kvm_api_version = 0;
1264
1265 sub kvm_version {
1266
1267     return $kvm_api_version if $kvm_api_version;
1268
1269     my $fh = IO::File->new("</dev/kvm") ||
1270         return 0;
1271
1272     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1273         $kvm_api_version = $v;
1274     }
1275
1276     $fh->close();
1277
1278     return  $kvm_api_version;
1279 }
1280
1281 my $kvm_user_version;
1282
1283 sub kvm_user_version {
1284
1285     return $kvm_user_version if $kvm_user_version;
1286
1287     $kvm_user_version = 'unknown';
1288
1289     my $code = sub {
1290         my $line = shift;
1291         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1292             $kvm_user_version = $2;
1293         }
1294     };
1295
1296     eval { run_command("kvm -version", outfunc => $code); };
1297     warn $@ if $@;
1298
1299     return $kvm_user_version;
1300
1301 }
1302
1303 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1304
1305 sub valid_drive_names {
1306     # order is important - used to autoselect boot disk
1307     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1308             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1309             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1310             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1311             'efidisk0');
1312 }
1313
1314 sub is_valid_drivename {
1315     my $dev = shift;
1316
1317     return defined($drivename_hash->{$dev});
1318 }
1319
1320 sub option_exists {
1321     my $key = shift;
1322     return defined($confdesc->{$key});
1323 }
1324
1325 sub nic_models {
1326     return $nic_model_list;
1327 }
1328
1329 sub os_list_description {
1330
1331     return {
1332         other => 'Other',
1333         wxp => 'Windows XP',
1334         w2k => 'Windows 2000',
1335         w2k3 =>, 'Windows 2003',
1336         w2k8 => 'Windows 2008',
1337         wvista => 'Windows Vista',
1338         win7 => 'Windows 7',
1339         win8 => 'Windows 8/2012',
1340         win10 => 'Windows 10/2016',
1341         l24 => 'Linux 2.4',
1342         l26 => 'Linux 2.6',
1343     };
1344 }
1345
1346 my $cdrom_path;
1347
1348 sub get_cdrom_path {
1349
1350     return  $cdrom_path if $cdrom_path;
1351
1352     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1353     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1354     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1355 }
1356
1357 sub get_iso_path {
1358     my ($storecfg, $vmid, $cdrom) = @_;
1359
1360     if ($cdrom eq 'cdrom') {
1361         return get_cdrom_path();
1362     } elsif ($cdrom eq 'none') {
1363         return '';
1364     } elsif ($cdrom =~ m|^/|) {
1365         return $cdrom;
1366     } else {
1367         return PVE::Storage::path($storecfg, $cdrom);
1368     }
1369 }
1370
1371 # try to convert old style file names to volume IDs
1372 sub filename_to_volume_id {
1373     my ($vmid, $file, $media) = @_;
1374
1375      if (!($file eq 'none' || $file eq 'cdrom' ||
1376           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1377
1378         return undef if $file =~ m|/|;
1379
1380         if ($media && $media eq 'cdrom') {
1381             $file = "local:iso/$file";
1382         } else {
1383             $file = "local:$vmid/$file";
1384         }
1385     }
1386
1387     return $file;
1388 }
1389
1390 sub verify_media_type {
1391     my ($opt, $vtype, $media) = @_;
1392
1393     return if !$media;
1394
1395     my $etype;
1396     if ($media eq 'disk') {
1397         $etype = 'images';
1398     } elsif ($media eq 'cdrom') {
1399         $etype = 'iso';
1400     } else {
1401         die "internal error";
1402     }
1403
1404     return if ($vtype eq $etype);
1405
1406     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1407 }
1408
1409 sub cleanup_drive_path {
1410     my ($opt, $storecfg, $drive) = @_;
1411
1412     # try to convert filesystem paths to volume IDs
1413
1414     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1415         ($drive->{file} !~ m|^/dev/.+|) &&
1416         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1417         ($drive->{file} !~ m/^\d+$/)) {
1418         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1419         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1420         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1421         verify_media_type($opt, $vtype, $drive->{media});
1422         $drive->{file} = $volid;
1423     }
1424
1425     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1426 }
1427
1428 sub parse_hotplug_features {
1429     my ($data) = @_;
1430
1431     my $res = {};
1432
1433     return $res if $data eq '0';
1434
1435     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1436
1437     foreach my $feature (PVE::Tools::split_list($data)) {
1438         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1439             $res->{$1} = 1;
1440         } else {
1441             die "invalid hotplug feature '$feature'\n";
1442         }
1443     }
1444     return $res;
1445 }
1446
1447 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1448 sub pve_verify_hotplug_features {
1449     my ($value, $noerr) = @_;
1450
1451     return $value if parse_hotplug_features($value);
1452
1453     return undef if $noerr;
1454
1455     die "unable to parse hotplug option\n";
1456 }
1457
1458 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1459 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1460 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1461 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1462 #        [,iothread=on][,serial=serial][,model=model]
1463
1464 sub parse_drive {
1465     my ($key, $data) = @_;
1466
1467     my ($interface, $index);
1468
1469     if ($key =~ m/^([^\d]+)(\d+)$/) {
1470         $interface = $1;
1471         $index = $2;
1472     } else {
1473         return undef;
1474     }
1475
1476     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1477                                      : $confdesc->{$key}->{format};
1478     if (!$desc) {
1479         warn "invalid drive key: $key\n";
1480         return undef;
1481     }
1482     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1483     return undef if !$res;
1484     $res->{interface} = $interface;
1485     $res->{index} = $index;
1486
1487     my $error = 0;
1488     foreach my $opt (qw(bps bps_rd bps_wr)) {
1489         if (my $bps = defined(delete $res->{$opt})) {
1490             if (defined($res->{"m$opt"})) {
1491                 warn "both $opt and m$opt specified\n";
1492                 ++$error;
1493                 next;
1494             }
1495             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1496         }
1497     }
1498
1499     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1500     for my $requirement (
1501         [mbps_max => 'mbps'],
1502         [mbps_rd_max => 'mbps_rd'],
1503         [mbps_wr_max => 'mbps_wr'],
1504         [miops_max => 'miops'],
1505         [miops_rd_max => 'miops_rd'],
1506         [miops_wr_max => 'miops_wr'],
1507         [bps_max_length => 'mbps_max'],
1508         [bps_rd_max_length => 'mbps_rd_max'],
1509         [bps_wr_max_length => 'mbps_wr_max'],
1510         [iops_max_length => 'iops_max'],
1511         [iops_rd_max_length => 'iops_rd_max'],
1512         [iops_wr_max_length => 'iops_wr_max']) {
1513         my ($option, $requires) = @$requirement;
1514         if ($res->{$option} && !$res->{$requires}) {
1515             warn "$option requires $requires\n";
1516             ++$error;
1517         }
1518     }
1519
1520     return undef if $error;
1521
1522     return undef if $res->{mbps_rd} && $res->{mbps};
1523     return undef if $res->{mbps_wr} && $res->{mbps};
1524     return undef if $res->{iops_rd} && $res->{iops};
1525     return undef if $res->{iops_wr} && $res->{iops};
1526
1527     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1528         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1529         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1530         return undef if $res->{interface} eq 'virtio';
1531     }
1532
1533     if (my $size = $res->{size}) {
1534         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1535     }
1536
1537     return $res;
1538 }
1539
1540 sub print_drive {
1541     my ($vmid, $drive) = @_;
1542     my $data = { %$drive };
1543     delete $data->{$_} for qw(index interface);
1544     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1545 }
1546
1547 sub scsi_inquiry {
1548     my($fh, $noerr) = @_;
1549
1550     my $SG_IO = 0x2285;
1551     my $SG_GET_VERSION_NUM = 0x2282;
1552
1553     my $versionbuf = "\x00" x 8;
1554     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1555     if (!$ret) {
1556         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1557         return undef;
1558     }
1559     my $version = unpack("I", $versionbuf);
1560     if ($version < 30000) {
1561         die "scsi generic interface too old\n"  if !$noerr;
1562         return undef;
1563     }
1564
1565     my $buf = "\x00" x 36;
1566     my $sensebuf = "\x00" x 8;
1567     my $cmd = pack("C x3 C x1", 0x12, 36);
1568
1569     # see /usr/include/scsi/sg.h
1570     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1571
1572     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1573                       length($sensebuf), 0, length($buf), $buf,
1574                       $cmd, $sensebuf, 6000);
1575
1576     $ret = ioctl($fh, $SG_IO, $packet);
1577     if (!$ret) {
1578         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1579         return undef;
1580     }
1581
1582     my @res = unpack($sg_io_hdr_t, $packet);
1583     if ($res[17] || $res[18]) {
1584         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1585         return undef;
1586     }
1587
1588     my $res = {};
1589     (my $byte0, my $byte1, $res->{vendor},
1590      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1591
1592     $res->{removable} = $byte1 & 128 ? 1 : 0;
1593     $res->{type} = $byte0 & 31;
1594
1595     return $res;
1596 }
1597
1598 sub path_is_scsi {
1599     my ($path) = @_;
1600
1601     my $fh = IO::File->new("+<$path") || return undef;
1602     my $res = scsi_inquiry($fh, 1);
1603     close($fh);
1604
1605     return $res;
1606 }
1607
1608 sub machine_type_is_q35 {
1609     my ($conf) = @_;
1610
1611     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1612 }
1613
1614 sub print_tabletdevice_full {
1615     my ($conf) = @_;
1616
1617     my $q35 = machine_type_is_q35($conf);
1618
1619     # we use uhci for old VMs because tablet driver was buggy in older qemu
1620     my $usbbus = $q35 ? "ehci" : "uhci";
1621
1622     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1623 }
1624
1625 sub print_drivedevice_full {
1626     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1627
1628     my $device = '';
1629     my $maxdev = 0;
1630
1631     if ($drive->{interface} eq 'virtio') {
1632         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1633         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1634         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1635     } elsif ($drive->{interface} eq 'scsi') {
1636
1637         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1638         my $unit = $drive->{index} % $maxdev;
1639         my $devicetype = 'hd';
1640         my $path = '';
1641         if (drive_is_cdrom($drive)) {
1642             $devicetype = 'cd';
1643         } else {
1644             if ($drive->{file} =~ m|^/|) {
1645                 $path = $drive->{file};
1646                 if (my $info = path_is_scsi($path)) {
1647                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1648                         $devicetype = 'block';
1649                     } elsif ($info->{type} == 1) { # tape
1650                         $devicetype = 'generic';
1651                     }
1652                 }
1653             } else {
1654                  $path = PVE::Storage::path($storecfg, $drive->{file});
1655             }
1656
1657             if($path =~ m/^iscsi\:\/\//){
1658                 $devicetype = 'generic';
1659             }
1660         }
1661
1662         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1663             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1664         } else {
1665             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1666         }
1667
1668     } elsif ($drive->{interface} eq 'ide'){
1669         $maxdev = 2;
1670         my $controller = int($drive->{index} / $maxdev);
1671         my $unit = $drive->{index} % $maxdev;
1672         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1673
1674         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1675         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1676             $model = URI::Escape::uri_unescape($model);
1677             $device .= ",model=$model";
1678         }
1679     } elsif ($drive->{interface} eq 'sata'){
1680         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1681         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1682         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1683     } elsif ($drive->{interface} eq 'usb') {
1684         die "implement me";
1685         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1686     } else {
1687         die "unsupported interface type";
1688     }
1689
1690     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1691
1692     if (my $serial = $drive->{serial}) {
1693         $serial = URI::Escape::uri_unescape($serial);
1694         $device .= ",serial=$serial";
1695     }
1696
1697
1698     return $device;
1699 }
1700
1701 sub get_initiator_name {
1702     my $initiator;
1703
1704     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1705     while (defined(my $line = <$fh>)) {
1706         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1707         $initiator = $1;
1708         last;
1709     }
1710     $fh->close();
1711
1712     return $initiator;
1713 }
1714
1715 sub print_drive_full {
1716     my ($storecfg, $vmid, $drive) = @_;
1717
1718     my $path;
1719     my $volid = $drive->{file};
1720     my $format;
1721
1722     if (drive_is_cdrom($drive)) {
1723         $path = get_iso_path($storecfg, $vmid, $volid);
1724     } else {
1725         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1726         if ($storeid) {
1727             $path = PVE::Storage::path($storecfg, $volid);
1728             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1729             $format = qemu_img_format($scfg, $volname);
1730         } else {
1731             $path = $volid;
1732             $format = "raw";
1733         }
1734    }
1735
1736     my $opts = '';
1737     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1738     foreach my $o (@qemu_drive_options) {
1739         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1740     }
1741
1742     # snapshot only accepts on|off
1743     if (defined($drive->{snapshot})) {
1744         my $v = $drive->{snapshot} ? 'on' : 'off';
1745         $opts .= ",snapshot=$v";
1746     }
1747
1748     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1749         my ($dir, $qmpname) = @$type;
1750         if (my $v = $drive->{"mbps$dir"}) {
1751             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1752         }
1753         if (my $v = $drive->{"mbps${dir}_max"}) {
1754             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1755         }
1756         if (my $v = $drive->{"bps${dir}_max_length"}) {
1757             $opts .= ",throttling.bps$qmpname-max-length=$v";
1758         }
1759         if (my $v = $drive->{"iops${dir}"}) {
1760             $opts .= ",throttling.iops$qmpname=$v";
1761         }
1762         if (my $v = $drive->{"iops${dir}_max"}) {
1763             $opts .= ",throttling.iops$qmpname-max=$v";
1764         }
1765         if (my $v = $drive->{"iops${dir}_max_length"}) {
1766             $opts .= ",throttling.iops$qmpname-max-length=$v";
1767         }
1768     }
1769
1770     $opts .= ",format=$format" if $format && !$drive->{format};
1771
1772     my $cache_direct = 0;
1773
1774     if (my $cache = $drive->{cache}) {
1775         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1776     } elsif (!drive_is_cdrom($drive)) {
1777         $opts .= ",cache=none";
1778         $cache_direct = 1;
1779     }
1780
1781     # aio native works only with O_DIRECT
1782     if (!$drive->{aio}) {
1783         if($cache_direct) {
1784             $opts .= ",aio=native";
1785         } else {
1786             $opts .= ",aio=threads";
1787         }
1788     }
1789
1790     if (!drive_is_cdrom($drive)) {
1791         my $detectzeroes;
1792         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1793             $detectzeroes = 'off';
1794         } elsif ($drive->{discard}) {
1795             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1796         } else {
1797             # This used to be our default with discard not being specified:
1798             $detectzeroes = 'on';
1799         }
1800         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1801     }
1802
1803     my $pathinfo = $path ? "file=$path," : '';
1804
1805     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1806 }
1807
1808 sub print_netdevice_full {
1809     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1810
1811     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1812
1813     my $device = $net->{model};
1814     if ($net->{model} eq 'virtio') {
1815          $device = 'virtio-net-pci';
1816      };
1817
1818     my $pciaddr = print_pci_addr("$netid", $bridges);
1819     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1820     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1821         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1822         my $vectors = $net->{queues} * 2 + 2;
1823         $tmpstr .= ",vectors=$vectors,mq=on";
1824     }
1825     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1826
1827     if ($use_old_bios_files) {
1828         my $romfile;
1829         if ($device eq 'virtio-net-pci') {
1830             $romfile = 'pxe-virtio.rom';
1831         } elsif ($device eq 'e1000') {
1832             $romfile = 'pxe-e1000.rom';
1833         } elsif ($device eq 'ne2k') {
1834             $romfile = 'pxe-ne2k_pci.rom';
1835         } elsif ($device eq 'pcnet') {
1836             $romfile = 'pxe-pcnet.rom';
1837         } elsif ($device eq 'rtl8139') {
1838             $romfile = 'pxe-rtl8139.rom';
1839         }
1840         $tmpstr .= ",romfile=$romfile" if $romfile;
1841     }
1842
1843     return $tmpstr;
1844 }
1845
1846 sub print_netdev_full {
1847     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1848
1849     my $i = '';
1850     if ($netid =~ m/^net(\d+)$/) {
1851         $i = int($1);
1852     }
1853
1854     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1855
1856     my $ifname = "tap${vmid}i$i";
1857
1858     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1859     die "interface name '$ifname' is too long (max 15 character)\n"
1860         if length($ifname) >= 16;
1861
1862     my $vhostparam = '';
1863     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1864
1865     my $vmname = $conf->{name} || "vm$vmid";
1866
1867     my $netdev = "";
1868     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1869
1870     if ($net->{bridge}) {
1871         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1872     } else {
1873         $netdev = "type=user,id=$netid,hostname=$vmname";
1874     }
1875
1876     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1877
1878     return $netdev;
1879 }
1880
1881
1882 sub print_cpu_device {
1883     my ($conf, $id) = @_;
1884
1885     my $kvm = $conf->{kvm} // 1;
1886     my $cpu = $kvm ? "kvm64" : "qemu64";
1887     if (my $cputype = $conf->{cpu}) {
1888         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1889             or die "Cannot parse cpu description: $cputype\n";
1890         $cpu = $cpuconf->{cputype};
1891     }
1892
1893     my $cores = $conf->{cores} || 1;
1894
1895     my $current_core = ($id - 1) % $cores;
1896     my $current_socket = int(($id - 1 - $current_core)/$cores);
1897
1898     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1899 }
1900
1901 sub drive_is_cloudinit {
1902     my ($drive) = @_;
1903     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1904 }
1905
1906 sub drive_is_cdrom {
1907     my ($drive, $exclude_cloudinit) = @_;
1908
1909     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1910
1911     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1912
1913 }
1914
1915 sub parse_number_sets {
1916     my ($set) = @_;
1917     my $res = [];
1918     foreach my $part (split(/;/, $set)) {
1919         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1920             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1921             push @$res, [ $1, $2 ];
1922         } else {
1923             die "invalid range: $part\n";
1924         }
1925     }
1926     return $res;
1927 }
1928
1929 sub parse_numa {
1930     my ($data) = @_;
1931
1932     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1933     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1934     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1935     return $res;
1936 }
1937
1938 sub parse_hostpci {
1939     my ($value) = @_;
1940
1941     return undef if !$value;
1942
1943     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1944
1945     my @idlist = split(/;/, $res->{host});
1946     delete $res->{host};
1947     foreach my $id (@idlist) {
1948         if ($id =~ /^$PCIRE$/) {
1949             if (defined($2)) {
1950                 push @{$res->{pciid}}, { id => $1, function => $2 };
1951             } else {
1952                 my $pcidevices = lspci($1);
1953                 $res->{pciid} = $pcidevices->{$1};
1954             }
1955         } else {
1956             # should have been caught by parse_property_string already
1957             die "failed to parse PCI id: $id\n";
1958         }
1959     }
1960     return $res;
1961 }
1962
1963 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1964 sub parse_net {
1965     my ($data) = @_;
1966
1967     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1968     if ($@) {
1969         warn $@;
1970         return undef;
1971     }
1972     if (!defined($res->{macaddr})) {
1973         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1974         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1975     }
1976     return $res;
1977 }
1978
1979 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1980 sub parse_ipconfig {
1981     my ($data) = @_;
1982
1983     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1984     if ($@) {
1985         warn $@;
1986         return undef;
1987     }
1988
1989     if ($res->{gw} && !$res->{ip}) {
1990         warn 'gateway specified without specifying an IP address';
1991         return undef;
1992     }
1993     if ($res->{gw6} && !$res->{ip6}) {
1994         warn 'IPv6 gateway specified without specifying an IPv6 address';
1995         return undef;
1996     }
1997     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1998         warn 'gateway specified together with DHCP';
1999         return undef;
2000     }
2001     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2002         # gw6 + auto/dhcp
2003         warn "IPv6 gateway specified together with $res->{ip6} address";
2004         return undef;
2005     }
2006
2007     if (!$res->{ip} && !$res->{ip6}) {
2008         return { ip => 'dhcp', ip6 => 'dhcp' };
2009     }
2010
2011     return $res;
2012 }
2013
2014 sub print_net {
2015     my $net = shift;
2016
2017     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2018 }
2019
2020 sub add_random_macs {
2021     my ($settings) = @_;
2022
2023     foreach my $opt (keys %$settings) {
2024         next if $opt !~ m/^net(\d+)$/;
2025         my $net = parse_net($settings->{$opt});
2026         next if !$net;
2027         $settings->{$opt} = print_net($net);
2028     }
2029 }
2030
2031 sub vm_is_volid_owner {
2032     my ($storecfg, $vmid, $volid) = @_;
2033
2034     if ($volid !~  m|^/|) {
2035         my ($path, $owner);
2036         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2037         if ($owner && ($owner == $vmid)) {
2038             return 1;
2039         }
2040     }
2041
2042     return undef;
2043 }
2044
2045 sub split_flagged_list {
2046     my $text = shift || '';
2047     $text =~ s/[,;]/ /g;
2048     $text =~ s/^\s+//;
2049     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2050 }
2051
2052 sub join_flagged_list {
2053     my ($how, $lst) = @_;
2054     join $how, map { $lst->{$_} . $_ } keys %$lst;
2055 }
2056
2057 sub vmconfig_delete_pending_option {
2058     my ($conf, $key, $force) = @_;
2059
2060     delete $conf->{pending}->{$key};
2061     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2062     $pending_delete_hash->{$key} = $force ? '!' : '';
2063     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2064 }
2065
2066 sub vmconfig_undelete_pending_option {
2067     my ($conf, $key) = @_;
2068
2069     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2070     delete $pending_delete_hash->{$key};
2071
2072     if (%$pending_delete_hash) {
2073         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2074     } else {
2075         delete $conf->{pending}->{delete};
2076     }
2077 }
2078
2079 sub vmconfig_register_unused_drive {
2080     my ($storecfg, $vmid, $conf, $drive) = @_;
2081
2082     if (drive_is_cloudinit($drive)) {
2083         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2084         warn $@ if $@;
2085     } elsif (!drive_is_cdrom($drive)) {
2086         my $volid = $drive->{file};
2087         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2088             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2089         }
2090     }
2091 }
2092
2093 sub vmconfig_cleanup_pending {
2094     my ($conf) = @_;
2095
2096     # remove pending changes when nothing changed
2097     my $changes;
2098     foreach my $opt (keys %{$conf->{pending}}) {
2099         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2100             $changes = 1;
2101             delete $conf->{pending}->{$opt};
2102         }
2103     }
2104
2105     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2106     my $pending_delete_hash = {};
2107     while (my ($opt, $force) = each %$current_delete_hash) {
2108         if (defined($conf->{$opt})) {
2109             $pending_delete_hash->{$opt} = $force;
2110         } else {
2111             $changes = 1;
2112         }
2113     }
2114
2115     if (%$pending_delete_hash) {
2116         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2117     } else {
2118         delete $conf->{pending}->{delete};
2119     }
2120
2121     return $changes;
2122 }
2123
2124 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2125 my $smbios1_fmt = {
2126     uuid => {
2127         type => 'string',
2128         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2129         format_description => 'UUID',
2130         description => "Set SMBIOS1 UUID.",
2131         optional => 1,
2132     },
2133     version => {
2134         type => 'string',
2135         pattern => '\S+',
2136         format_description => 'string',
2137         description => "Set SMBIOS1 version.",
2138         optional => 1,
2139     },
2140     serial => {
2141         type => 'string',
2142         pattern => '\S+',
2143         format_description => 'string',
2144         description => "Set SMBIOS1 serial number.",
2145         optional => 1,
2146     },
2147     manufacturer => {
2148         type => 'string',
2149         pattern => '\S+',
2150         format_description => 'string',
2151         description => "Set SMBIOS1 manufacturer.",
2152         optional => 1,
2153     },
2154     product => {
2155         type => 'string',
2156         pattern => '\S+',
2157         format_description => 'string',
2158         description => "Set SMBIOS1 product ID.",
2159         optional => 1,
2160     },
2161     sku => {
2162         type => 'string',
2163         pattern => '\S+',
2164         format_description => 'string',
2165         description => "Set SMBIOS1 SKU string.",
2166         optional => 1,
2167     },
2168     family => {
2169         type => 'string',
2170         pattern => '\S+',
2171         format_description => 'string',
2172         description => "Set SMBIOS1 family string.",
2173         optional => 1,
2174     },
2175 };
2176
2177 sub parse_smbios1 {
2178     my ($data) = @_;
2179
2180     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2181     warn $@ if $@;
2182     return $res;
2183 }
2184
2185 sub print_smbios1 {
2186     my ($smbios1) = @_;
2187     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2188 }
2189
2190 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2191
2192 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2193 sub verify_bootdisk {
2194     my ($value, $noerr) = @_;
2195
2196     return $value if is_valid_drivename($value);
2197
2198     return undef if $noerr;
2199
2200     die "invalid boot disk '$value'\n";
2201 }
2202
2203 sub parse_watchdog {
2204     my ($value) = @_;
2205
2206     return undef if !$value;
2207
2208     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2209     warn $@ if $@;
2210     return $res;
2211 }
2212
2213 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2214 sub verify_usb_device {
2215     my ($value, $noerr) = @_;
2216
2217     return $value if parse_usb_device($value);
2218
2219     return undef if $noerr;
2220
2221     die "unable to parse usb device\n";
2222 }
2223
2224 # add JSON properties for create and set function
2225 sub json_config_properties {
2226     my $prop = shift;
2227
2228     foreach my $opt (keys %$confdesc) {
2229         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2230         $prop->{$opt} = $confdesc->{$opt};
2231     }
2232
2233     return $prop;
2234 }
2235
2236 # return copy of $confdesc_cloudinit to generate documentation
2237 sub cloudinit_config_properties {
2238
2239     return dclone($confdesc_cloudinit);
2240 }
2241
2242 sub check_type {
2243     my ($key, $value) = @_;
2244
2245     die "unknown setting '$key'\n" if !$confdesc->{$key};
2246
2247     my $type = $confdesc->{$key}->{type};
2248
2249     if (!defined($value)) {
2250         die "got undefined value\n";
2251     }
2252
2253     if ($value =~ m/[\n\r]/) {
2254         die "property contains a line feed\n";
2255     }
2256
2257     if ($type eq 'boolean') {
2258         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2259         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2260         die "type check ('boolean') failed - got '$value'\n";
2261     } elsif ($type eq 'integer') {
2262         return int($1) if $value =~ m/^(\d+)$/;
2263         die "type check ('integer') failed - got '$value'\n";
2264     } elsif ($type eq 'number') {
2265         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2266         die "type check ('number') failed - got '$value'\n";
2267     } elsif ($type eq 'string') {
2268         if (my $fmt = $confdesc->{$key}->{format}) {
2269             PVE::JSONSchema::check_format($fmt, $value);
2270             return $value;
2271         }
2272         $value =~ s/^\"(.*)\"$/$1/;
2273         return $value;
2274     } else {
2275         die "internal error"
2276     }
2277 }
2278
2279 sub check_iommu_support{
2280     #fixme : need to check IOMMU support
2281     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2282
2283     my $iommu=1;
2284     return $iommu;
2285
2286 }
2287
2288 sub touch_config {
2289     my ($vmid) = @_;
2290
2291     my $conf = PVE::QemuConfig->config_file($vmid);
2292     utime undef, undef, $conf;
2293 }
2294
2295 sub destroy_vm {
2296     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2297
2298     my $conffile = PVE::QemuConfig->config_file($vmid);
2299
2300     my $conf = PVE::QemuConfig->load_config($vmid);
2301
2302     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2303
2304     if ($conf->{template}) {
2305         # check if any base image is still used by a linked clone
2306         foreach_drive($conf, sub {
2307                 my ($ds, $drive) = @_;
2308
2309                 return if drive_is_cdrom($drive);
2310
2311                 my $volid = $drive->{file};
2312
2313                 return if !$volid || $volid =~ m|^/|;
2314
2315                 die "base volume '$volid' is still in use by linked cloned\n"
2316                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2317
2318         });
2319     }
2320
2321     # only remove disks owned by this VM
2322     foreach_drive($conf, sub {
2323         my ($ds, $drive) = @_;
2324
2325         return if drive_is_cdrom($drive, 1);
2326
2327         my $volid = $drive->{file};
2328
2329         return if !$volid || $volid =~ m|^/|;
2330
2331         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2332         return if !$path || !$owner || ($owner != $vmid);
2333
2334         eval {
2335             PVE::Storage::vdisk_free($storecfg, $volid);
2336         };
2337         warn "Could not remove disk '$volid', check manually: $@" if $@;
2338
2339     });
2340
2341     if ($keep_empty_config) {
2342         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2343     } else {
2344         unlink $conffile;
2345     }
2346
2347     # also remove unused disk
2348     eval {
2349         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2350
2351         eval {
2352             PVE::Storage::foreach_volid($dl, sub {
2353                 my ($volid, $sid, $volname, $d) = @_;
2354                 PVE::Storage::vdisk_free($storecfg, $volid);
2355             });
2356         };
2357         warn $@ if $@;
2358
2359     };
2360     warn $@ if $@;
2361 }
2362
2363 sub parse_vm_config {
2364     my ($filename, $raw) = @_;
2365
2366     return undef if !defined($raw);
2367
2368     my $res = {
2369         digest => Digest::SHA::sha1_hex($raw),
2370         snapshots => {},
2371         pending => {},
2372     };
2373
2374     $filename =~ m|/qemu-server/(\d+)\.conf$|
2375         || die "got strange filename '$filename'";
2376
2377     my $vmid = $1;
2378
2379     my $conf = $res;
2380     my $descr;
2381     my $section = '';
2382
2383     my @lines = split(/\n/, $raw);
2384     foreach my $line (@lines) {
2385         next if $line =~ m/^\s*$/;
2386
2387         if ($line =~ m/^\[PENDING\]\s*$/i) {
2388             $section = 'pending';
2389             if (defined($descr)) {
2390                 $descr =~ s/\s+$//;
2391                 $conf->{description} = $descr;
2392             }
2393             $descr = undef;
2394             $conf = $res->{$section} = {};
2395             next;
2396
2397         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2398             $section = $1;
2399             if (defined($descr)) {
2400                 $descr =~ s/\s+$//;
2401                 $conf->{description} = $descr;
2402             }
2403             $descr = undef;
2404             $conf = $res->{snapshots}->{$section} = {};
2405             next;
2406         }
2407
2408         if ($line =~ m/^\#(.*)\s*$/) {
2409             $descr = '' if !defined($descr);
2410             $descr .= PVE::Tools::decode_text($1) . "\n";
2411             next;
2412         }
2413
2414         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2415             $descr = '' if !defined($descr);
2416             $descr .= PVE::Tools::decode_text($2);
2417         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2418             $conf->{snapstate} = $1;
2419         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2420             my $key = $1;
2421             my $value = $2;
2422             $conf->{$key} = $value;
2423         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2424             my $value = $1;
2425             if ($section eq 'pending') {
2426                 $conf->{delete} = $value; # we parse this later
2427             } else {
2428                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2429             }
2430         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2431             my $key = $1;
2432             my $value = $2;
2433             eval { $value = check_type($key, $value); };
2434             if ($@) {
2435                 warn "vm $vmid - unable to parse value of '$key' - $@";
2436             } else {
2437                 $key = 'ide2' if $key eq 'cdrom';
2438                 my $fmt = $confdesc->{$key}->{format};
2439                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2440                     my $v = parse_drive($key, $value);
2441                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2442                         $v->{file} = $volid;
2443                         $value = print_drive($vmid, $v);
2444                     } else {
2445                         warn "vm $vmid - unable to parse value of '$key'\n";
2446                         next;
2447                     }
2448                 }
2449
2450                 $conf->{$key} = $value;
2451             }
2452         }
2453     }
2454
2455     if (defined($descr)) {
2456         $descr =~ s/\s+$//;
2457         $conf->{description} = $descr;
2458     }
2459     delete $res->{snapstate}; # just to be sure
2460
2461     return $res;
2462 }
2463
2464 sub write_vm_config {
2465     my ($filename, $conf) = @_;
2466
2467     delete $conf->{snapstate}; # just to be sure
2468
2469     if ($conf->{cdrom}) {
2470         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2471         $conf->{ide2} = $conf->{cdrom};
2472         delete $conf->{cdrom};
2473     }
2474
2475     # we do not use 'smp' any longer
2476     if ($conf->{sockets}) {
2477         delete $conf->{smp};
2478     } elsif ($conf->{smp}) {
2479         $conf->{sockets} = $conf->{smp};
2480         delete $conf->{cores};
2481         delete $conf->{smp};
2482     }
2483
2484     my $used_volids = {};
2485
2486     my $cleanup_config = sub {
2487         my ($cref, $pending, $snapname) = @_;
2488
2489         foreach my $key (keys %$cref) {
2490             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2491                 $key eq 'snapstate' || $key eq 'pending';
2492             my $value = $cref->{$key};
2493             if ($key eq 'delete') {
2494                 die "propertry 'delete' is only allowed in [PENDING]\n"
2495                     if !$pending;
2496                 # fixme: check syntax?
2497                 next;
2498             }
2499             eval { $value = check_type($key, $value); };
2500             die "unable to parse value of '$key' - $@" if $@;
2501
2502             $cref->{$key} = $value;
2503
2504             if (!$snapname && is_valid_drivename($key)) {
2505                 my $drive = parse_drive($key, $value);
2506                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2507             }
2508         }
2509     };
2510
2511     &$cleanup_config($conf);
2512
2513     &$cleanup_config($conf->{pending}, 1);
2514
2515     foreach my $snapname (keys %{$conf->{snapshots}}) {
2516         die "internal error" if $snapname eq 'pending';
2517         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2518     }
2519
2520     # remove 'unusedX' settings if we re-add a volume
2521     foreach my $key (keys %$conf) {
2522         my $value = $conf->{$key};
2523         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2524             delete $conf->{$key};
2525         }
2526     }
2527
2528     my $generate_raw_config = sub {
2529         my ($conf, $pending) = @_;
2530
2531         my $raw = '';
2532
2533         # add description as comment to top of file
2534         if (defined(my $descr = $conf->{description})) {
2535             if ($descr) {
2536                 foreach my $cl (split(/\n/, $descr)) {
2537                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2538                 }
2539             } else {
2540                 $raw .= "#\n" if $pending;
2541             }
2542         }
2543
2544         foreach my $key (sort keys %$conf) {
2545             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2546             $raw .= "$key: $conf->{$key}\n";
2547         }
2548         return $raw;
2549     };
2550
2551     my $raw = &$generate_raw_config($conf);
2552
2553     if (scalar(keys %{$conf->{pending}})){
2554         $raw .= "\n[PENDING]\n";
2555         $raw .= &$generate_raw_config($conf->{pending}, 1);
2556     }
2557
2558     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2559         $raw .= "\n[$snapname]\n";
2560         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2561     }
2562
2563     return $raw;
2564 }
2565
2566 sub load_defaults {
2567
2568     my $res = {};
2569
2570     # we use static defaults from our JSON schema configuration
2571     foreach my $key (keys %$confdesc) {
2572         if (defined(my $default = $confdesc->{$key}->{default})) {
2573             $res->{$key} = $default;
2574         }
2575     }
2576
2577     return $res;
2578 }
2579
2580 sub config_list {
2581     my $vmlist = PVE::Cluster::get_vmlist();
2582     my $res = {};
2583     return $res if !$vmlist || !$vmlist->{ids};
2584     my $ids = $vmlist->{ids};
2585
2586     foreach my $vmid (keys %$ids) {
2587         my $d = $ids->{$vmid};
2588         next if !$d->{node} || $d->{node} ne $nodename;
2589         next if !$d->{type} || $d->{type} ne 'qemu';
2590         $res->{$vmid}->{exists} = 1;
2591     }
2592     return $res;
2593 }
2594
2595 # test if VM uses local resources (to prevent migration)
2596 sub check_local_resources {
2597     my ($conf, $noerr) = @_;
2598
2599     my $loc_res = 0;
2600
2601     $loc_res = 1 if $conf->{hostusb}; # old syntax
2602     $loc_res = 1 if $conf->{hostpci}; # old syntax
2603
2604     foreach my $k (keys %$conf) {
2605         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2606         # sockets are safe: they will recreated be on the target side post-migrate
2607         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2608         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2609     }
2610
2611     die "VM uses local resources\n" if $loc_res && !$noerr;
2612
2613     return $loc_res;
2614 }
2615
2616 # check if used storages are available on all nodes (use by migrate)
2617 sub check_storage_availability {
2618     my ($storecfg, $conf, $node) = @_;
2619
2620     foreach_drive($conf, sub {
2621         my ($ds, $drive) = @_;
2622
2623         my $volid = $drive->{file};
2624         return if !$volid;
2625
2626         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2627         return if !$sid;
2628
2629         # check if storage is available on both nodes
2630         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2631         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2632    });
2633 }
2634
2635 # list nodes where all VM images are available (used by has_feature API)
2636 sub shared_nodes {
2637     my ($conf, $storecfg) = @_;
2638
2639     my $nodelist = PVE::Cluster::get_nodelist();
2640     my $nodehash = { map { $_ => 1 } @$nodelist };
2641     my $nodename = PVE::INotify::nodename();
2642
2643     foreach_drive($conf, sub {
2644         my ($ds, $drive) = @_;
2645
2646         my $volid = $drive->{file};
2647         return if !$volid;
2648
2649         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2650         if ($storeid) {
2651             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2652             if ($scfg->{disable}) {
2653                 $nodehash = {};
2654             } elsif (my $avail = $scfg->{nodes}) {
2655                 foreach my $node (keys %$nodehash) {
2656                     delete $nodehash->{$node} if !$avail->{$node};
2657                 }
2658             } elsif (!$scfg->{shared}) {
2659                 foreach my $node (keys %$nodehash) {
2660                     delete $nodehash->{$node} if $node ne $nodename
2661                 }
2662             }
2663         }
2664     });
2665
2666     return $nodehash
2667 }
2668
2669 sub check_cmdline {
2670     my ($pidfile, $pid) = @_;
2671
2672     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2673     if (defined($fh)) {
2674         my $line = <$fh>;
2675         $fh->close;
2676         return undef if !$line;
2677         my @param = split(/\0/, $line);
2678
2679         my $cmd = $param[0];
2680         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2681
2682         for (my $i = 0; $i < scalar (@param); $i++) {
2683             my $p = $param[$i];
2684             next if !$p;
2685             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2686                 my $p = $param[$i+1];
2687                 return 1 if $p && ($p eq $pidfile);
2688                 return undef;
2689             }
2690         }
2691     }
2692     return undef;
2693 }
2694
2695 sub check_running {
2696     my ($vmid, $nocheck, $node) = @_;
2697
2698     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2699
2700     die "unable to find configuration file for VM $vmid - no such machine\n"
2701         if !$nocheck && ! -f $filename;
2702
2703     my $pidfile = pidfile_name($vmid);
2704
2705     if (my $fd = IO::File->new("<$pidfile")) {
2706         my $st = stat($fd);
2707         my $line = <$fd>;
2708         close($fd);
2709
2710         my $mtime = $st->mtime;
2711         if ($mtime > time()) {
2712             warn "file '$filename' modified in future\n";
2713         }
2714
2715         if ($line =~ m/^(\d+)$/) {
2716             my $pid = $1;
2717             if (check_cmdline($pidfile, $pid)) {
2718                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2719                     return $pid;
2720                 }
2721             }
2722         }
2723     }
2724
2725     return undef;
2726 }
2727
2728 sub vzlist {
2729
2730     my $vzlist = config_list();
2731
2732     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2733
2734     while (defined(my $de = $fd->read)) {
2735         next if $de !~ m/^(\d+)\.pid$/;
2736         my $vmid = $1;
2737         next if !defined($vzlist->{$vmid});
2738         if (my $pid = check_running($vmid)) {
2739             $vzlist->{$vmid}->{pid} = $pid;
2740         }
2741     }
2742
2743     return $vzlist;
2744 }
2745
2746 sub disksize {
2747     my ($storecfg, $conf) = @_;
2748
2749     my $bootdisk = $conf->{bootdisk};
2750     return undef if !$bootdisk;
2751     return undef if !is_valid_drivename($bootdisk);
2752
2753     return undef if !$conf->{$bootdisk};
2754
2755     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2756     return undef if !defined($drive);
2757
2758     return undef if drive_is_cdrom($drive);
2759
2760     my $volid = $drive->{file};
2761     return undef if !$volid;
2762
2763     return $drive->{size};
2764 }
2765
2766 my $last_proc_pid_stat;
2767
2768 # get VM status information
2769 # This must be fast and should not block ($full == false)
2770 # We only query KVM using QMP if $full == true (this can be slow)
2771 sub vmstatus {
2772     my ($opt_vmid, $full) = @_;
2773
2774     my $res = {};
2775
2776     my $storecfg = PVE::Storage::config();
2777
2778     my $list = vzlist();
2779     my $defaults = load_defaults();
2780
2781     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2782
2783     my $cpucount = $cpuinfo->{cpus} || 1;
2784
2785     foreach my $vmid (keys %$list) {
2786         next if $opt_vmid && ($vmid ne $opt_vmid);
2787
2788         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2789         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2790
2791         my $d = {};
2792         $d->{pid} = $list->{$vmid}->{pid};
2793
2794         # fixme: better status?
2795         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2796
2797         my $size = disksize($storecfg, $conf);
2798         if (defined($size)) {
2799             $d->{disk} = 0; # no info available
2800             $d->{maxdisk} = $size;
2801         } else {
2802             $d->{disk} = 0;
2803             $d->{maxdisk} = 0;
2804         }
2805
2806         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2807             * ($conf->{cores} || $defaults->{cores});
2808         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2809         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2810
2811         $d->{name} = $conf->{name} || "VM $vmid";
2812         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2813             : $defaults->{memory}*(1024*1024);
2814
2815         if ($conf->{balloon}) {
2816             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2817             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2818                 : $defaults->{shares};
2819         }
2820
2821         $d->{uptime} = 0;
2822         $d->{cpu} = 0;
2823         $d->{mem} = 0;
2824
2825         $d->{netout} = 0;
2826         $d->{netin} = 0;
2827
2828         $d->{diskread} = 0;
2829         $d->{diskwrite} = 0;
2830
2831         $d->{template} = PVE::QemuConfig->is_template($conf);
2832
2833         $d->{serial} = 1 if conf_has_serial($conf);
2834
2835         $res->{$vmid} = $d;
2836     }
2837
2838     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2839     foreach my $dev (keys %$netdev) {
2840         next if $dev !~ m/^tap([1-9]\d*)i/;
2841         my $vmid = $1;
2842         my $d = $res->{$vmid};
2843         next if !$d;
2844
2845         $d->{netout} += $netdev->{$dev}->{receive};
2846         $d->{netin} += $netdev->{$dev}->{transmit};
2847
2848         if ($full) {
2849             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2850             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2851         }
2852
2853     }
2854
2855     my $ctime = gettimeofday;
2856
2857     foreach my $vmid (keys %$list) {
2858
2859         my $d = $res->{$vmid};
2860         my $pid = $d->{pid};
2861         next if !$pid;
2862
2863         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2864         next if !$pstat; # not running
2865
2866         my $used = $pstat->{utime} + $pstat->{stime};
2867
2868         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2869
2870         if ($pstat->{vsize}) {
2871             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2872         }
2873
2874         my $old = $last_proc_pid_stat->{$pid};
2875         if (!$old) {
2876             $last_proc_pid_stat->{$pid} = {
2877                 time => $ctime,
2878                 used => $used,
2879                 cpu => 0,
2880             };
2881             next;
2882         }
2883
2884         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2885
2886         if ($dtime > 1000) {
2887             my $dutime = $used -  $old->{used};
2888
2889             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2890             $last_proc_pid_stat->{$pid} = {
2891                 time => $ctime,
2892                 used => $used,
2893                 cpu => $d->{cpu},
2894             };
2895         } else {
2896             $d->{cpu} = $old->{cpu};
2897         }
2898     }
2899
2900     return $res if !$full;
2901
2902     my $qmpclient = PVE::QMPClient->new();
2903
2904     my $ballooncb = sub {
2905         my ($vmid, $resp) = @_;
2906
2907         my $info = $resp->{'return'};
2908         return if !$info->{max_mem};
2909
2910         my $d = $res->{$vmid};
2911
2912         # use memory assigned to VM
2913         $d->{maxmem} = $info->{max_mem};
2914         $d->{balloon} = $info->{actual};
2915
2916         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2917             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2918             $d->{freemem} = $info->{free_mem};
2919         }
2920
2921         $d->{ballooninfo} = $info;
2922     };
2923
2924     my $blockstatscb = sub {
2925         my ($vmid, $resp) = @_;
2926         my $data = $resp->{'return'} || [];
2927         my $totalrdbytes = 0;
2928         my $totalwrbytes = 0;
2929
2930         for my $blockstat (@$data) {
2931             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2932             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2933
2934             $blockstat->{device} =~ s/drive-//;
2935             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2936         }
2937         $res->{$vmid}->{diskread} = $totalrdbytes;
2938         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2939     };
2940
2941     my $statuscb = sub {
2942         my ($vmid, $resp) = @_;
2943
2944         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2945         # this fails if ballon driver is not loaded, so this must be
2946         # the last commnand (following command are aborted if this fails).
2947         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2948
2949         my $status = 'unknown';
2950         if (!defined($status = $resp->{'return'}->{status})) {
2951             warn "unable to get VM status\n";
2952             return;
2953         }
2954
2955         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2956     };
2957
2958     foreach my $vmid (keys %$list) {
2959         next if $opt_vmid && ($vmid ne $opt_vmid);
2960         next if !$res->{$vmid}->{pid}; # not running
2961         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2962     }
2963
2964     $qmpclient->queue_execute(undef, 2);
2965
2966     foreach my $vmid (keys %$list) {
2967         next if $opt_vmid && ($vmid ne $opt_vmid);
2968         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2969     }
2970
2971     return $res;
2972 }
2973
2974 sub foreach_drive {
2975     my ($conf, $func, @param) = @_;
2976
2977     foreach my $ds (valid_drive_names()) {
2978         next if !defined($conf->{$ds});
2979
2980         my $drive = parse_drive($ds, $conf->{$ds});
2981         next if !$drive;
2982
2983         &$func($ds, $drive, @param);
2984     }
2985 }
2986
2987 sub foreach_volid {
2988     my ($conf, $func, @param) = @_;
2989
2990     my $volhash = {};
2991
2992     my $test_volid = sub {
2993         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2994
2995         return if !$volid;
2996
2997         $volhash->{$volid}->{cdrom} //= 1;
2998         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2999
3000         $volhash->{$volid}->{replicate} //= 0;
3001         $volhash->{$volid}->{replicate} = 1 if $replicate;
3002
3003         $volhash->{$volid}->{shared} //= 0;
3004         $volhash->{$volid}->{shared} = 1 if $shared;
3005
3006         $volhash->{$volid}->{referenced_in_config} //= 0;
3007         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3008
3009         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3010             if defined($snapname);
3011     };
3012
3013     foreach_drive($conf, sub {
3014         my ($ds, $drive) = @_;
3015         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3016     });
3017
3018     foreach my $snapname (keys %{$conf->{snapshots}}) {
3019         my $snap = $conf->{snapshots}->{$snapname};
3020         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3021         foreach_drive($snap, sub {
3022             my ($ds, $drive) = @_;
3023             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3024         });
3025     }
3026
3027     foreach my $volid (keys %$volhash) {
3028         &$func($volid, $volhash->{$volid}, @param);
3029     }
3030 }
3031
3032 sub conf_has_serial {
3033     my ($conf) = @_;
3034
3035     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3036         if ($conf->{"serial$i"}) {
3037             return 1;
3038         }
3039     }
3040
3041     return 0;
3042 }
3043
3044 sub vga_conf_has_spice {
3045     my ($vga) = @_;
3046
3047     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3048
3049     return $1 || 1;
3050 }
3051
3052 sub config_to_command {
3053     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3054
3055     my $cmd = [];
3056     my $globalFlags = [];
3057     my $machineFlags = [];
3058     my $rtcFlags = [];
3059     my $cpuFlags = [];
3060     my $devices = [];
3061     my $pciaddr = '';
3062     my $bridges = {};
3063     my $kvmver = kvm_user_version();
3064     my $vernum = 0; # unknown
3065     my $ostype = $conf->{ostype};
3066     my $winversion = windows_version($ostype);
3067     my $kvm = $conf->{kvm} // 1;
3068
3069     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3070
3071     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3072         $vernum = $1*1000000+$2*1000;
3073     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3074         $vernum = $1*1000000+$2*1000+$3;
3075     }
3076
3077     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3078
3079     my $have_ovz = -f '/proc/vz/vestat';
3080
3081     my $q35 = machine_type_is_q35($conf);
3082     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3083     my $machine_type = $forcemachine || $conf->{machine};
3084     my $use_old_bios_files = undef;
3085     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3086
3087     my $cpuunits = defined($conf->{cpuunits}) ?
3088             $conf->{cpuunits} : $defaults->{cpuunits};
3089
3090     push @$cmd, '/usr/bin/kvm';
3091
3092     push @$cmd, '-id', $vmid;
3093
3094     my $vmname = $conf->{name} || "vm$vmid";
3095
3096     push @$cmd, '-name', $vmname;
3097
3098     my $use_virtio = 0;
3099
3100     my $qmpsocket = qmp_socket($vmid);
3101     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3102     push @$cmd, '-mon', "chardev=qmp,mode=control";
3103
3104
3105     push @$cmd, '-pidfile' , pidfile_name($vmid);
3106
3107     push @$cmd, '-daemonize';
3108
3109     if ($conf->{smbios1}) {
3110         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3111     }
3112
3113     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3114         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3115
3116         my $path;
3117         my $format;
3118         if (my $efidisk = $conf->{efidisk0}) {
3119             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3120             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3121             $format = $d->{format};
3122             if ($storeid) {
3123                 $path = PVE::Storage::path($storecfg, $d->{file});
3124                 if (!defined($format)) {
3125                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3126                     $format = qemu_img_format($scfg, $volname);
3127                 }
3128             } else {
3129                 $path = $d->{file};
3130                 die "efidisk format must be specified\n"
3131                     if !defined($format);
3132             }
3133         } else {
3134             warn "no efidisk configured! Using temporary efivars disk.\n";
3135             $path = "/tmp/$vmid-ovmf.fd";
3136             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3137             $format = 'raw';
3138         }
3139
3140         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3141         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3142     }
3143
3144
3145     # add usb controllers
3146     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3147     push @$devices, @usbcontrollers if @usbcontrollers;
3148     my $vga = $conf->{vga};
3149
3150     my $qxlnum = vga_conf_has_spice($vga);
3151     $vga = 'qxl' if $qxlnum;
3152
3153     if (!$vga) {
3154         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3155             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3156         } else {
3157             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3158         }
3159     }
3160
3161     # enable absolute mouse coordinates (needed by vnc)
3162     my $tablet;
3163     if (defined($conf->{tablet})) {
3164         $tablet = $conf->{tablet};
3165     } else {
3166         $tablet = $defaults->{tablet};
3167         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3168         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3169     }
3170
3171     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3172
3173     my $kvm_off = 0;
3174     my $gpu_passthrough;
3175
3176     # host pci devices
3177     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3178         my $d = parse_hostpci($conf->{"hostpci$i"});
3179         next if !$d;
3180
3181         my $pcie = $d->{pcie};
3182         if($pcie){
3183             die "q35 machine model is not enabled" if !$q35;
3184             $pciaddr = print_pcie_addr("hostpci$i");
3185         }else{
3186             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3187         }
3188
3189         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3190         my $romfile = $d->{romfile};
3191
3192         my $xvga = '';
3193         if ($d->{'x-vga'}) {
3194             $xvga = ',x-vga=on';
3195             $kvm_off = 1;
3196             $vga = 'none';
3197             $gpu_passthrough = 1;
3198
3199             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3200                 $xvga = "";
3201             }
3202         }
3203         my $pcidevices = $d->{pciid};
3204         my $multifunction = 1 if @$pcidevices > 1;
3205
3206         my $j=0;
3207         foreach my $pcidevice (@$pcidevices) {
3208
3209             my $id = "hostpci$i";
3210             $id .= ".$j" if $multifunction;
3211             my $addr = $pciaddr;
3212             $addr .= ".$j" if $multifunction;
3213             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3214
3215             if($j == 0){
3216                 $devicestr .= "$rombar$xvga";
3217                 $devicestr .= ",multifunction=on" if $multifunction;
3218                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3219             }
3220
3221             push @$devices, '-device', $devicestr;
3222             $j++;
3223         }
3224     }
3225
3226     # usb devices
3227     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3228     push @$devices, @usbdevices if @usbdevices;
3229     # serial devices
3230     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3231         if (my $path = $conf->{"serial$i"}) {
3232             if ($path eq 'socket') {
3233                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3234                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3235                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3236             } else {
3237                 die "no such serial device\n" if ! -c $path;
3238                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3239                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3240             }
3241         }
3242     }
3243
3244     # parallel devices
3245     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3246         if (my $path = $conf->{"parallel$i"}) {
3247             die "no such parallel device\n" if ! -c $path;
3248             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3249             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3250             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3251         }
3252     }
3253
3254
3255     my $sockets = 1;
3256     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3257     $sockets = $conf->{sockets} if  $conf->{sockets};
3258
3259     my $cores = $conf->{cores} || 1;
3260
3261     my $maxcpus = $sockets * $cores;
3262
3263     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3264
3265     my $allowed_vcpus = $cpuinfo->{cpus};
3266
3267     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3268         if ($allowed_vcpus < $maxcpus);
3269
3270     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3271
3272         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3273         for (my $i = 2; $i <= $vcpus; $i++)  {
3274             my $cpustr = print_cpu_device($conf,$i);
3275             push @$cmd, '-device', $cpustr;
3276         }
3277
3278     } else {
3279
3280         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3281     }
3282     push @$cmd, '-nodefaults';
3283
3284     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3285
3286     my $bootindex_hash = {};
3287     my $i = 1;
3288     foreach my $o (split(//, $bootorder)) {
3289         $bootindex_hash->{$o} = $i*100;
3290         $i++;
3291     }
3292
3293     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3294
3295     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3296
3297     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3298
3299     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3300
3301     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3302         my $socket = vnc_socket($vmid);
3303         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3304     } else {
3305         push @$cmd, '-nographic';
3306     }
3307
3308     # time drift fix
3309     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3310
3311     my $useLocaltime = $conf->{localtime};
3312
3313     if ($winversion >= 5) { # windows
3314         $useLocaltime = 1 if !defined($conf->{localtime});
3315
3316         # use time drift fix when acpi is enabled
3317         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3318             $tdf = 1 if !defined($conf->{tdf});
3319         }
3320     }
3321
3322     if ($winversion >= 6) {
3323         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3324         push @$cmd, '-no-hpet';
3325     }
3326
3327     push @$rtcFlags, 'driftfix=slew' if $tdf;
3328
3329     if (!$kvm) {
3330         push @$machineFlags, 'accel=tcg';
3331     }
3332
3333     if ($machine_type) {
3334         push @$machineFlags, "type=${machine_type}";
3335     }
3336
3337     if ($conf->{startdate}) {
3338         push @$rtcFlags, "base=$conf->{startdate}";
3339     } elsif ($useLocaltime) {
3340         push @$rtcFlags, 'base=localtime';
3341     }
3342
3343     my $cpu = $kvm ? "kvm64" : "qemu64";
3344     if (my $cputype = $conf->{cpu}) {
3345         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3346             or die "Cannot parse cpu description: $cputype\n";
3347         $cpu = $cpuconf->{cputype};
3348         $kvm_off = 1 if $cpuconf->{hidden};
3349
3350         if (defined(my $flags = $cpuconf->{flags})) {
3351             push @$cpuFlags, split(";", $flags);
3352         }
3353     }
3354
3355     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3356
3357     push @$cpuFlags , '-x2apic'
3358         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3359
3360     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3361
3362     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3363
3364     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3365
3366         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3367         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3368     }
3369
3370     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3371
3372     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3373
3374     push @$cpuFlags, 'kvm=off' if $kvm_off;
3375
3376     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3377         die "internal error"; # should not happen
3378
3379     push @$cpuFlags, "vendor=${cpu_vendor}"
3380         if $cpu_vendor ne 'default';
3381
3382     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3383
3384     push @$cmd, '-cpu', $cpu;
3385
3386     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3387
3388     push @$cmd, '-S' if $conf->{freeze};
3389
3390     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3391
3392     # enable sound
3393     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3394     #push @$cmd, '-soundhw', 'es1370';
3395     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3396
3397     if($conf->{agent}) {
3398         my $qgasocket = qmp_socket($vmid, 1);
3399         my $pciaddr = print_pci_addr("qga0", $bridges);
3400         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3401         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3402         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3403     }
3404
3405     my $spice_port;
3406
3407     if ($qxlnum) {
3408         if ($qxlnum > 1) {
3409             if ($winversion){
3410                 for(my $i = 1; $i < $qxlnum; $i++){
3411                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3412                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3413                 }
3414             } else {
3415                 # assume other OS works like Linux
3416                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3417                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3418             }
3419         }
3420
3421         my $pciaddr = print_pci_addr("spice", $bridges);
3422
3423         my $nodename = PVE::INotify::nodename();
3424         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3425         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3426         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3427         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3428         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3429
3430         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3431
3432         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3433         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3434         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3435     }
3436
3437     # enable balloon by default, unless explicitly disabled
3438     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3439         $pciaddr = print_pci_addr("balloon0", $bridges);
3440         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3441     }
3442
3443     if ($conf->{watchdog}) {
3444         my $wdopts = parse_watchdog($conf->{watchdog});
3445         $pciaddr = print_pci_addr("watchdog", $bridges);
3446         my $watchdog = $wdopts->{model} || 'i6300esb';
3447         push @$devices, '-device', "$watchdog$pciaddr";
3448         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3449     }
3450
3451     my $vollist = [];
3452     my $scsicontroller = {};
3453     my $ahcicontroller = {};
3454     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3455
3456     # Add iscsi initiator name if available
3457     if (my $initiator = get_initiator_name()) {
3458         push @$devices, '-iscsi', "initiator-name=$initiator";
3459     }
3460
3461     foreach_drive($conf, sub {
3462         my ($ds, $drive) = @_;
3463
3464         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3465             push @$vollist, $drive->{file};
3466         }
3467
3468         # ignore efidisk here, already added in bios/fw handling code above
3469         return if $drive->{interface} eq 'efidisk';
3470
3471         $use_virtio = 1 if $ds =~ m/^virtio/;
3472
3473         if (drive_is_cdrom ($drive)) {
3474             if ($bootindex_hash->{d}) {
3475                 $drive->{bootindex} = $bootindex_hash->{d};
3476                 $bootindex_hash->{d} += 1;
3477             }
3478         } else {
3479             if ($bootindex_hash->{c}) {
3480                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3481                 $bootindex_hash->{c} += 1;
3482             }
3483         }
3484
3485         if($drive->{interface} eq 'virtio'){
3486            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3487         }
3488
3489         if ($drive->{interface} eq 'scsi') {
3490
3491             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3492
3493             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3494             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3495
3496             my $iothread = '';
3497             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3498                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3499                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3500             } elsif ($drive->{iothread}) {
3501                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3502             }
3503
3504             my $queues = '';
3505             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3506                 $queues = ",num_queues=$drive->{queues}";
3507             }
3508
3509             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3510             $scsicontroller->{$controller}=1;
3511         }
3512
3513         if ($drive->{interface} eq 'sata') {
3514            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3515            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3516            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3517            $ahcicontroller->{$controller}=1;
3518         }
3519
3520         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3521         push @$devices, '-drive',$drive_cmd;
3522         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3523     });
3524
3525     for (my $i = 0; $i < $MAX_NETS; $i++) {
3526          next if !$conf->{"net$i"};
3527          my $d = parse_net($conf->{"net$i"});
3528          next if !$d;
3529
3530          $use_virtio = 1 if $d->{model} eq 'virtio';
3531
3532          if ($bootindex_hash->{n}) {
3533             $d->{bootindex} = $bootindex_hash->{n};
3534             $bootindex_hash->{n} += 1;
3535          }
3536
3537          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3538          push @$devices, '-netdev', $netdevfull;
3539
3540          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3541          push @$devices, '-device', $netdevicefull;
3542     }
3543
3544     if (!$q35) {
3545         # add pci bridges
3546         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3547            $bridges->{1} = 1;
3548            $bridges->{2} = 1;
3549         }
3550
3551         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3552
3553         while (my ($k, $v) = each %$bridges) {
3554             $pciaddr = print_pci_addr("pci.$k");
3555             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3556         }
3557     }
3558
3559     # add custom args
3560     if ($conf->{args}) {
3561         my $aa = PVE::Tools::split_args($conf->{args});
3562         push @$cmd, @$aa;
3563     }
3564
3565     push @$cmd, @$devices;
3566     push @$cmd, '-rtc', join(',', @$rtcFlags)
3567         if scalar(@$rtcFlags);
3568     push @$cmd, '-machine', join(',', @$machineFlags)
3569         if scalar(@$machineFlags);
3570     push @$cmd, '-global', join(',', @$globalFlags)
3571         if scalar(@$globalFlags);
3572
3573     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3574 }
3575
3576 sub vnc_socket {
3577     my ($vmid) = @_;
3578     return "${var_run_tmpdir}/$vmid.vnc";
3579 }
3580
3581 sub spice_port {
3582     my ($vmid) = @_;
3583
3584     my $res = vm_mon_cmd($vmid, 'query-spice');
3585
3586     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3587 }
3588
3589 sub qmp_socket {
3590     my ($vmid, $qga) = @_;
3591     my $sockettype = $qga ? 'qga' : 'qmp';
3592     return "${var_run_tmpdir}/$vmid.$sockettype";
3593 }
3594
3595 sub pidfile_name {
3596     my ($vmid) = @_;
3597     return "${var_run_tmpdir}/$vmid.pid";
3598 }
3599
3600 sub vm_devices_list {
3601     my ($vmid) = @_;
3602
3603     my $res = vm_mon_cmd($vmid, 'query-pci');
3604     my $devices_to_check = [];
3605     my $devices = {};
3606     foreach my $pcibus (@$res) {
3607         push @$devices_to_check, @{$pcibus->{devices}},
3608     }
3609
3610     while (@$devices_to_check) {
3611         my $to_check = [];
3612         for my $d (@$devices_to_check) {
3613             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3614             next if !$d->{'pci_bridge'};
3615
3616             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3617             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3618         }
3619         $devices_to_check = $to_check;
3620     }
3621
3622     my $resblock = vm_mon_cmd($vmid, 'query-block');
3623     foreach my $block (@$resblock) {
3624         if($block->{device} =~ m/^drive-(\S+)/){
3625                 $devices->{$1} = 1;
3626         }
3627     }
3628
3629     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3630     foreach my $mice (@$resmice) {
3631         if ($mice->{name} eq 'QEMU HID Tablet') {
3632             $devices->{tablet} = 1;
3633             last;
3634         }
3635     }
3636
3637     # for usb devices there is no query-usb
3638     # but we can iterate over the entries in
3639     # qom-list path=/machine/peripheral
3640     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3641     foreach my $per (@$resperipheral) {
3642         if ($per->{name} =~ m/^usb\d+$/) {
3643             $devices->{$per->{name}} = 1;
3644         }
3645     }
3646
3647     return $devices;
3648 }
3649
3650 sub vm_deviceplug {
3651     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3652
3653     my $q35 = machine_type_is_q35($conf);
3654
3655     my $devices_list = vm_devices_list($vmid);
3656     return 1 if defined($devices_list->{$deviceid});
3657
3658     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3659
3660     if ($deviceid eq 'tablet') {
3661
3662         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3663
3664     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3665
3666         die "usb hotplug currently not reliable\n";
3667         # since we can't reliably hot unplug all added usb devices
3668         # and usb passthrough disables live migration
3669         # we disable usb hotplugging for now
3670         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3671
3672     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3673
3674         qemu_iothread_add($vmid, $deviceid, $device);
3675
3676         qemu_driveadd($storecfg, $vmid, $device);
3677         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3678
3679         qemu_deviceadd($vmid, $devicefull);
3680         eval { qemu_deviceaddverify($vmid, $deviceid); };
3681         if (my $err = $@) {
3682             eval { qemu_drivedel($vmid, $deviceid); };
3683             warn $@ if $@;
3684             die $err;
3685         }
3686
3687     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3688
3689
3690         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3691         my $pciaddr = print_pci_addr($deviceid);
3692         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3693
3694         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3695
3696         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3697             qemu_iothread_add($vmid, $deviceid, $device);
3698             $devicefull .= ",iothread=iothread-$deviceid";
3699         }
3700
3701         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3702             $devicefull .= ",num_queues=$device->{queues}";
3703         }
3704
3705         qemu_deviceadd($vmid, $devicefull);
3706         qemu_deviceaddverify($vmid, $deviceid);
3707
3708     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3709
3710         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3711         qemu_driveadd($storecfg, $vmid, $device);
3712
3713         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3714         eval { qemu_deviceadd($vmid, $devicefull); };
3715         if (my $err = $@) {
3716             eval { qemu_drivedel($vmid, $deviceid); };
3717             warn $@ if $@;
3718             die $err;
3719         }
3720
3721     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3722
3723         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3724
3725         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3726         my $use_old_bios_files = undef;
3727         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3728
3729         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3730         qemu_deviceadd($vmid, $netdevicefull);
3731         eval { qemu_deviceaddverify($vmid, $deviceid); };
3732         if (my $err = $@) {
3733             eval { qemu_netdevdel($vmid, $deviceid); };
3734             warn $@ if $@;
3735             die $err;
3736         }
3737
3738     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3739
3740         my $bridgeid = $2;
3741         my $pciaddr = print_pci_addr($deviceid);
3742         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3743
3744         qemu_deviceadd($vmid, $devicefull);
3745         qemu_deviceaddverify($vmid, $deviceid);
3746
3747     } else {
3748         die "can't hotplug device '$deviceid'\n";
3749     }
3750
3751     return 1;
3752 }
3753
3754 # fixme: this should raise exceptions on error!
3755 sub vm_deviceunplug {
3756     my ($vmid, $conf, $deviceid) = @_;
3757
3758     my $devices_list = vm_devices_list($vmid);
3759     return 1 if !defined($devices_list->{$deviceid});
3760
3761     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3762
3763     if ($deviceid eq 'tablet') {
3764
3765         qemu_devicedel($vmid, $deviceid);
3766
3767     } elsif ($deviceid =~ m/^usb\d+$/) {
3768
3769         die "usb hotplug currently not reliable\n";
3770         # when unplugging usb devices this way,
3771         # there may be remaining usb controllers/hubs
3772         # so we disable it for now
3773         qemu_devicedel($vmid, $deviceid);
3774         qemu_devicedelverify($vmid, $deviceid);
3775
3776     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3777
3778         qemu_devicedel($vmid, $deviceid);
3779         qemu_devicedelverify($vmid, $deviceid);
3780         qemu_drivedel($vmid, $deviceid);
3781         qemu_iothread_del($conf, $vmid, $deviceid);
3782
3783     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3784
3785         qemu_devicedel($vmid, $deviceid);
3786         qemu_devicedelverify($vmid, $deviceid);
3787         qemu_iothread_del($conf, $vmid, $deviceid);
3788
3789     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3790
3791         qemu_devicedel($vmid, $deviceid);
3792         qemu_drivedel($vmid, $deviceid);
3793         qemu_deletescsihw($conf, $vmid, $deviceid);
3794
3795     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3796
3797         qemu_devicedel($vmid, $deviceid);
3798         qemu_devicedelverify($vmid, $deviceid);
3799         qemu_netdevdel($vmid, $deviceid);
3800
3801     } else {
3802         die "can't unplug device '$deviceid'\n";
3803     }
3804
3805     return 1;
3806 }
3807
3808 sub qemu_deviceadd {
3809     my ($vmid, $devicefull) = @_;
3810
3811     $devicefull = "driver=".$devicefull;
3812     my %options =  split(/[=,]/, $devicefull);
3813
3814     vm_mon_cmd($vmid, "device_add" , %options);
3815 }
3816
3817 sub qemu_devicedel {
3818     my ($vmid, $deviceid) = @_;
3819
3820     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3821 }
3822
3823 sub qemu_iothread_add {
3824     my($vmid, $deviceid, $device) = @_;
3825
3826     if ($device->{iothread}) {
3827         my $iothreads = vm_iothreads_list($vmid);
3828         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3829     }
3830 }
3831
3832 sub qemu_iothread_del {
3833     my($conf, $vmid, $deviceid) = @_;
3834
3835     my $device = parse_drive($deviceid, $conf->{$deviceid});
3836     if ($device->{iothread}) {
3837         my $iothreads = vm_iothreads_list($vmid);
3838         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3839     }
3840 }
3841
3842 sub qemu_objectadd {
3843     my($vmid, $objectid, $qomtype) = @_;
3844
3845     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3846
3847     return 1;
3848 }
3849
3850 sub qemu_objectdel {
3851     my($vmid, $objectid) = @_;
3852
3853     vm_mon_cmd($vmid, "object-del", id => $objectid);
3854
3855     return 1;
3856 }
3857
3858 sub qemu_driveadd {
3859     my ($storecfg, $vmid, $device) = @_;
3860
3861     my $drive = print_drive_full($storecfg, $vmid, $device);
3862     $drive =~ s/\\/\\\\/g;
3863     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3864
3865     # If the command succeeds qemu prints: "OK"
3866     return 1 if $ret =~ m/OK/s;
3867
3868     die "adding drive failed: $ret\n";
3869 }
3870
3871 sub qemu_drivedel {
3872     my($vmid, $deviceid) = @_;
3873
3874     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3875     $ret =~ s/^\s+//;
3876
3877     return 1 if $ret eq "";
3878
3879     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3880     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3881
3882     die "deleting drive $deviceid failed : $ret\n";
3883 }
3884
3885 sub qemu_deviceaddverify {
3886     my ($vmid, $deviceid) = @_;
3887
3888     for (my $i = 0; $i <= 5; $i++) {
3889          my $devices_list = vm_devices_list($vmid);
3890          return 1 if defined($devices_list->{$deviceid});
3891          sleep 1;
3892     }
3893
3894     die "error on hotplug device '$deviceid'\n";
3895 }
3896
3897
3898 sub qemu_devicedelverify {
3899     my ($vmid, $deviceid) = @_;
3900
3901     # need to verify that the device is correctly removed as device_del
3902     # is async and empty return is not reliable
3903
3904     for (my $i = 0; $i <= 5; $i++) {
3905          my $devices_list = vm_devices_list($vmid);
3906          return 1 if !defined($devices_list->{$deviceid});
3907          sleep 1;
3908     }
3909
3910     die "error on hot-unplugging device '$deviceid'\n";
3911 }
3912
3913 sub qemu_findorcreatescsihw {
3914     my ($storecfg, $conf, $vmid, $device) = @_;
3915
3916     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3917
3918     my $scsihwid="$controller_prefix$controller";
3919     my $devices_list = vm_devices_list($vmid);
3920
3921     if(!defined($devices_list->{$scsihwid})) {
3922         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3923     }
3924
3925     return 1;
3926 }
3927
3928 sub qemu_deletescsihw {
3929     my ($conf, $vmid, $opt) = @_;
3930
3931     my $device = parse_drive($opt, $conf->{$opt});
3932
3933     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3934         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3935         return 1;
3936     }
3937
3938     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3939
3940     my $devices_list = vm_devices_list($vmid);
3941     foreach my $opt (keys %{$devices_list}) {
3942         if (PVE::QemuServer::is_valid_drivename($opt)) {
3943             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3944             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3945                 return 1;
3946             }
3947         }
3948     }
3949
3950     my $scsihwid="scsihw$controller";
3951
3952     vm_deviceunplug($vmid, $conf, $scsihwid);
3953
3954     return 1;
3955 }
3956
3957 sub qemu_add_pci_bridge {
3958     my ($storecfg, $conf, $vmid, $device) = @_;
3959
3960     my $bridges = {};
3961
3962     my $bridgeid;
3963
3964     print_pci_addr($device, $bridges);
3965
3966     while (my ($k, $v) = each %$bridges) {
3967         $bridgeid = $k;
3968     }
3969     return 1 if !defined($bridgeid) || $bridgeid < 1;
3970
3971     my $bridge = "pci.$bridgeid";
3972     my $devices_list = vm_devices_list($vmid);
3973
3974     if (!defined($devices_list->{$bridge})) {
3975         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3976     }
3977
3978     return 1;
3979 }
3980
3981 sub qemu_set_link_status {
3982     my ($vmid, $device, $up) = @_;
3983
3984     vm_mon_cmd($vmid, "set_link", name => $device,
3985                up => $up ? JSON::true : JSON::false);
3986 }
3987
3988 sub qemu_netdevadd {
3989     my ($vmid, $conf, $device, $deviceid) = @_;
3990
3991     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3992     my %options =  split(/[=,]/, $netdev);
3993
3994     vm_mon_cmd($vmid, "netdev_add",  %options);
3995     return 1;
3996 }
3997
3998 sub qemu_netdevdel {
3999     my ($vmid, $deviceid) = @_;
4000
4001     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4002 }
4003
4004 sub qemu_usb_hotplug {
4005     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4006
4007     return if !$device;
4008
4009     # remove the old one first
4010     vm_deviceunplug($vmid, $conf, $deviceid);
4011
4012     # check if xhci controller is necessary and available
4013     if ($device->{usb3}) {
4014
4015         my $devicelist = vm_devices_list($vmid);
4016
4017         if (!$devicelist->{xhci}) {
4018             my $pciaddr = print_pci_addr("xhci");
4019             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4020         }
4021     }
4022     my $d = parse_usb_device($device->{host});
4023     $d->{usb3} = $device->{usb3};
4024
4025     # add the new one
4026     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4027 }
4028
4029 sub qemu_cpu_hotplug {
4030     my ($vmid, $conf, $vcpus) = @_;
4031
4032     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4033
4034     my $sockets = 1;
4035     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4036     $sockets = $conf->{sockets} if  $conf->{sockets};
4037     my $cores = $conf->{cores} || 1;
4038     my $maxcpus = $sockets * $cores;
4039
4040     $vcpus = $maxcpus if !$vcpus;
4041
4042     die "you can't add more vcpus than maxcpus\n"
4043         if $vcpus > $maxcpus;
4044
4045     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4046
4047     if ($vcpus < $currentvcpus) {
4048
4049         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4050
4051             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4052                 qemu_devicedel($vmid, "cpu$i");
4053                 my $retry = 0;
4054                 my $currentrunningvcpus = undef;
4055                 while (1) {
4056                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4057                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4058                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4059                     $retry++;
4060                     sleep 1;
4061                 }
4062                 #update conf after each succesfull cpu unplug
4063                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4064                 PVE::QemuConfig->write_config($vmid, $conf);
4065             }
4066         } else {
4067             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4068         }
4069
4070         return;
4071     }
4072
4073     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4074     die "vcpus in running vm does not match its configuration\n"
4075         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4076
4077     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4078
4079         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4080             my $cpustr = print_cpu_device($conf, $i);
4081             qemu_deviceadd($vmid, $cpustr);
4082
4083             my $retry = 0;
4084             my $currentrunningvcpus = undef;
4085             while (1) {
4086                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4087                 last if scalar(@{$currentrunningvcpus}) == $i;
4088                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4089                 sleep 1;
4090                 $retry++;
4091             }
4092             #update conf after each succesfull cpu hotplug
4093             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4094             PVE::QemuConfig->write_config($vmid, $conf);
4095         }
4096     } else {
4097
4098         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4099             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4100         }
4101     }
4102 }
4103
4104 sub qemu_block_set_io_throttle {
4105     my ($vmid, $deviceid,
4106         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4107         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4108         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4109         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4110
4111     return if !check_running($vmid) ;
4112
4113     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4114         bps => int($bps),
4115         bps_rd => int($bps_rd),
4116         bps_wr => int($bps_wr),
4117         iops => int($iops),
4118         iops_rd => int($iops_rd),
4119         iops_wr => int($iops_wr),
4120         bps_max => int($bps_max),
4121         bps_rd_max => int($bps_rd_max),
4122         bps_wr_max => int($bps_wr_max),
4123         iops_max => int($iops_max),
4124         iops_rd_max => int($iops_rd_max),
4125         iops_wr_max => int($iops_wr_max),
4126         bps_max_length => int($bps_max_length),
4127         bps_rd_max_length => int($bps_rd_max_length),
4128         bps_wr_max_length => int($bps_wr_max_length),
4129         iops_max_length => int($iops_max_length),
4130         iops_rd_max_length => int($iops_rd_max_length),
4131         iops_wr_max_length => int($iops_wr_max_length),
4132     );
4133
4134 }
4135
4136 # old code, only used to shutdown old VM after update
4137 sub __read_avail {
4138     my ($fh, $timeout) = @_;
4139
4140     my $sel = new IO::Select;
4141     $sel->add($fh);
4142
4143     my $res = '';
4144     my $buf;
4145
4146     my @ready;
4147     while (scalar (@ready = $sel->can_read($timeout))) {
4148         my $count;
4149         if ($count = $fh->sysread($buf, 8192)) {
4150             if ($buf =~ /^(.*)\(qemu\) $/s) {
4151                 $res .= $1;
4152                 last;
4153             } else {
4154                 $res .= $buf;
4155             }
4156         } else {
4157             if (!defined($count)) {
4158                 die "$!\n";
4159             }
4160             last;
4161         }
4162     }
4163
4164     die "monitor read timeout\n" if !scalar(@ready);
4165
4166     return $res;
4167 }
4168
4169 sub qemu_block_resize {
4170     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4171
4172     my $running = check_running($vmid);
4173
4174     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4175
4176     return if !$running;
4177
4178     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4179
4180 }
4181
4182 sub qemu_volume_snapshot {
4183     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4184
4185     my $running = check_running($vmid);
4186
4187     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4188         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4189     } else {
4190         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4191     }
4192 }
4193
4194 sub qemu_volume_snapshot_delete {
4195     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4196
4197     my $running = check_running($vmid);
4198
4199     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4200         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4201     } else {
4202         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4203     }
4204 }
4205
4206 sub set_migration_caps {
4207     my ($vmid) = @_;
4208
4209     my $cap_ref = [];
4210
4211     my $enabled_cap = {
4212         "auto-converge" => 1,
4213         "xbzrle" => 1,
4214         "x-rdma-pin-all" => 0,
4215         "zero-blocks" => 0,
4216         "compress" => 0
4217     };
4218
4219     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4220
4221     for my $supported_capability (@$supported_capabilities) {
4222         push @$cap_ref, {
4223             capability => $supported_capability->{capability},
4224             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4225         };
4226     }
4227
4228     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4229 }
4230
4231 my $fast_plug_option = {
4232     'lock' => 1,
4233     'name' => 1,
4234     'onboot' => 1,
4235     'shares' => 1,
4236     'startup' => 1,
4237     'description' => 1,
4238     'protection' => 1,
4239     'vmstatestorage' => 1,
4240 };
4241
4242 # hotplug changes in [PENDING]
4243 # $selection hash can be used to only apply specified options, for
4244 # example: { cores => 1 } (only apply changed 'cores')
4245 # $errors ref is used to return error messages
4246 sub vmconfig_hotplug_pending {
4247     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4248
4249     my $defaults = load_defaults();
4250
4251     # commit values which do not have any impact on running VM first
4252     # Note: those option cannot raise errors, we we do not care about
4253     # $selection and always apply them.
4254
4255     my $add_error = sub {
4256         my ($opt, $msg) = @_;
4257         $errors->{$opt} = "hotplug problem - $msg";
4258     };
4259
4260     my $changes = 0;
4261     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4262         if ($fast_plug_option->{$opt}) {
4263             $conf->{$opt} = $conf->{pending}->{$opt};
4264             delete $conf->{pending}->{$opt};
4265             $changes = 1;
4266         }
4267     }
4268
4269     if ($changes) {
4270         PVE::QemuConfig->write_config($vmid, $conf);
4271         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4272     }
4273
4274     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4275
4276     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4277     while (my ($opt, $force) = each %$pending_delete_hash) {
4278         next if $selection && !$selection->{$opt};
4279         eval {
4280             if ($opt eq 'hotplug') {
4281                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4282             } elsif ($opt eq 'tablet') {
4283                 die "skip\n" if !$hotplug_features->{usb};
4284                 if ($defaults->{tablet}) {
4285                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4286                 } else {
4287                     vm_deviceunplug($vmid, $conf, $opt);
4288                 }
4289             } elsif ($opt =~ m/^usb\d+/) {
4290                 die "skip\n";
4291                 # since we cannot reliably hot unplug usb devices
4292                 # we are disabling it
4293                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4294                 vm_deviceunplug($vmid, $conf, $opt);
4295             } elsif ($opt eq 'vcpus') {
4296                 die "skip\n" if !$hotplug_features->{cpu};
4297                 qemu_cpu_hotplug($vmid, $conf, undef);
4298             } elsif ($opt eq 'balloon') {
4299                 # enable balloon device is not hotpluggable
4300                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4301             } elsif ($fast_plug_option->{$opt}) {
4302                 # do nothing
4303             } elsif ($opt =~ m/^net(\d+)$/) {
4304                 die "skip\n" if !$hotplug_features->{network};
4305                 vm_deviceunplug($vmid, $conf, $opt);
4306             } elsif (is_valid_drivename($opt)) {
4307                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4308                 vm_deviceunplug($vmid, $conf, $opt);
4309                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4310             } elsif ($opt =~ m/^memory$/) {
4311                 die "skip\n" if !$hotplug_features->{memory};
4312                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4313             } elsif ($opt eq 'cpuunits') {
4314                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4315             } elsif ($opt eq 'cpulimit') {
4316                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4317             } else {
4318                 die "skip\n";
4319             }
4320         };
4321         if (my $err = $@) {
4322             &$add_error($opt, $err) if $err ne "skip\n";
4323         } else {
4324             # save new config if hotplug was successful
4325             delete $conf->{$opt};
4326             vmconfig_undelete_pending_option($conf, $opt);
4327             PVE::QemuConfig->write_config($vmid, $conf);
4328             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4329         }
4330     }
4331
4332     my $apply_pending_cloudinit;
4333     $apply_pending_cloudinit = sub {
4334         my ($key, $value) = @_;
4335         $apply_pending_cloudinit = sub {}; # once is enough
4336
4337         my @cloudinit_opts = keys %$confdesc_cloudinit;
4338         foreach my $opt (keys %{$conf->{pending}}) {
4339             next if !grep { $_ eq $opt } @cloudinit_opts;
4340             $conf->{$opt} = delete $conf->{pending}->{$opt};
4341         }
4342
4343         my $new_conf = { %$conf };
4344         $new_conf->{$key} = $value;
4345         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4346     };
4347
4348     foreach my $opt (keys %{$conf->{pending}}) {
4349         next if $selection && !$selection->{$opt};
4350         my $value = $conf->{pending}->{$opt};
4351         eval {
4352             if ($opt eq 'hotplug') {
4353                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4354             } elsif ($opt eq 'tablet') {
4355                 die "skip\n" if !$hotplug_features->{usb};
4356                 if ($value == 1) {
4357                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4358                 } elsif ($value == 0) {
4359                     vm_deviceunplug($vmid, $conf, $opt);
4360                 }
4361             } elsif ($opt =~ m/^usb\d+$/) {
4362                 die "skip\n";
4363                 # since we cannot reliably hot unplug usb devices
4364                 # we are disabling it
4365                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4366                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4367                 die "skip\n" if !$d;
4368                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4369             } elsif ($opt eq 'vcpus') {
4370                 die "skip\n" if !$hotplug_features->{cpu};
4371                 qemu_cpu_hotplug($vmid, $conf, $value);
4372             } elsif ($opt eq 'balloon') {
4373                 # enable/disable balloning device is not hotpluggable
4374                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4375                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4376                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4377
4378                 # allow manual ballooning if shares is set to zero
4379                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4380                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4381                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4382                 }
4383             } elsif ($opt =~ m/^net(\d+)$/) {
4384                 # some changes can be done without hotplug
4385                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4386                                     $vmid, $opt, $value);
4387             } elsif (is_valid_drivename($opt)) {
4388                 # some changes can be done without hotplug
4389                 my $drive = parse_drive($opt, $value);
4390                 if (drive_is_cloudinit($drive)) {
4391                     &$apply_pending_cloudinit($opt, $value);
4392                 }
4393                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4394                                      $vmid, $opt, $value, 1);
4395             } elsif ($opt =~ m/^memory$/) { #dimms
4396                 die "skip\n" if !$hotplug_features->{memory};
4397                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4398             } elsif ($opt eq 'cpuunits') {
4399                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4400             } elsif ($opt eq 'cpulimit') {
4401                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4402                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4403             } else {
4404                 die "skip\n";  # skip non-hot-pluggable options
4405             }
4406         };
4407         if (my $err = $@) {
4408             &$add_error($opt, $err) if $err ne "skip\n";
4409         } else {
4410             # save new config if hotplug was successful
4411             $conf->{$opt} = $value;
4412             delete $conf->{pending}->{$opt};
4413             PVE::QemuConfig->write_config($vmid, $conf);
4414             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4415         }
4416     }
4417 }
4418
4419 sub try_deallocate_drive {
4420     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4421
4422     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4423         my $volid = $drive->{file};
4424         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4425             my $sid = PVE::Storage::parse_volume_id($volid);
4426             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4427
4428             # check if the disk is really unused
4429             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4430                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4431             PVE::Storage::vdisk_free($storecfg, $volid);
4432             return 1;
4433         } else {
4434             # If vm is not owner of this disk remove from config
4435             return 1;
4436         }
4437     }
4438
4439     return undef;
4440 }
4441
4442 sub vmconfig_delete_or_detach_drive {
4443     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4444
4445     my $drive = parse_drive($opt, $conf->{$opt});
4446
4447     my $rpcenv = PVE::RPCEnvironment::get();
4448     my $authuser = $rpcenv->get_user();
4449
4450     if ($force) {
4451         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4452         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4453     } else {
4454         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4455     }
4456 }
4457
4458 sub vmconfig_apply_pending {
4459     my ($vmid, $conf, $storecfg) = @_;
4460
4461     # cold plug
4462
4463     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4464     while (my ($opt, $force) = each %$pending_delete_hash) {
4465         die "internal error" if $opt =~ m/^unused/;
4466         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4467         if (!defined($conf->{$opt})) {
4468             vmconfig_undelete_pending_option($conf, $opt);
4469             PVE::QemuConfig->write_config($vmid, $conf);
4470         } elsif (is_valid_drivename($opt)) {
4471             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4472             vmconfig_undelete_pending_option($conf, $opt);
4473             delete $conf->{$opt};
4474             PVE::QemuConfig->write_config($vmid, $conf);
4475         } else {
4476             vmconfig_undelete_pending_option($conf, $opt);
4477             delete $conf->{$opt};
4478             PVE::QemuConfig->write_config($vmid, $conf);
4479         }
4480     }
4481
4482     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4483
4484     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4485         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4486
4487         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4488             # skip if nothing changed
4489         } elsif (is_valid_drivename($opt)) {
4490             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4491                 if defined($conf->{$opt});
4492             $conf->{$opt} = $conf->{pending}->{$opt};
4493         } else {
4494             $conf->{$opt} = $conf->{pending}->{$opt};
4495         }
4496
4497         delete $conf->{pending}->{$opt};
4498         PVE::QemuConfig->write_config($vmid, $conf);
4499     }
4500 }
4501
4502 my $safe_num_ne = sub {
4503     my ($a, $b) = @_;
4504
4505     return 0 if !defined($a) && !defined($b);
4506     return 1 if !defined($a);
4507     return 1 if !defined($b);
4508
4509     return $a != $b;
4510 };
4511
4512 my $safe_string_ne = sub {
4513     my ($a, $b) = @_;
4514
4515     return 0 if !defined($a) && !defined($b);
4516     return 1 if !defined($a);
4517     return 1 if !defined($b);
4518
4519     return $a ne $b;
4520 };
4521
4522 sub vmconfig_update_net {
4523     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4524
4525     my $newnet = parse_net($value);
4526
4527     if ($conf->{$opt}) {
4528         my $oldnet = parse_net($conf->{$opt});
4529
4530         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4531             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4532             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4533             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4534
4535             # for non online change, we try to hot-unplug
4536             die "skip\n" if !$hotplug;
4537             vm_deviceunplug($vmid, $conf, $opt);
4538         } else {
4539
4540             die "internal error" if $opt !~ m/net(\d+)/;
4541             my $iface = "tap${vmid}i$1";
4542
4543             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4544                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4545                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4546                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4547                 PVE::Network::tap_unplug($iface);
4548                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4549             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4550                 # Rate can be applied on its own but any change above needs to
4551                 # include the rate in tap_plug since OVS resets everything.
4552                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4553             }
4554
4555             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4556                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4557             }
4558
4559             return 1;
4560         }
4561     }
4562
4563     if ($hotplug) {
4564         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4565     } else {
4566         die "skip\n";
4567     }
4568 }
4569
4570 sub vmconfig_update_disk {
4571     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4572
4573     # fixme: do we need force?
4574
4575     my $drive = parse_drive($opt, $value);
4576
4577     if ($conf->{$opt}) {
4578
4579         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4580
4581             my $media = $drive->{media} || 'disk';
4582             my $oldmedia = $old_drive->{media} || 'disk';
4583             die "unable to change media type\n" if $media ne $oldmedia;
4584
4585             if (!drive_is_cdrom($old_drive)) {
4586
4587                 if ($drive->{file} ne $old_drive->{file}) {
4588
4589                     die "skip\n" if !$hotplug;
4590
4591                     # unplug and register as unused
4592                     vm_deviceunplug($vmid, $conf, $opt);
4593                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4594
4595                 } else {
4596                     # update existing disk
4597
4598                     # skip non hotpluggable value
4599                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4600                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4601                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4602                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4603                         die "skip\n";
4604                     }
4605
4606                     # apply throttle
4607                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4608                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4609                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4610                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4611                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4612                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4613                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4614                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4615                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4616                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4617                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4618                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4619                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4620                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4621                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4622                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4623                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4624                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4625
4626                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4627                                                    ($drive->{mbps} || 0)*1024*1024,
4628                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4629                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4630                                                    $drive->{iops} || 0,
4631                                                    $drive->{iops_rd} || 0,
4632                                                    $drive->{iops_wr} || 0,
4633                                                    ($drive->{mbps_max} || 0)*1024*1024,
4634                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4635                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4636                                                    $drive->{iops_max} || 0,
4637                                                    $drive->{iops_rd_max} || 0,
4638                                                    $drive->{iops_wr_max} || 0,
4639                                                    $drive->{bps_max_length} || 1,
4640                                                    $drive->{bps_rd_max_length} || 1,
4641                                                    $drive->{bps_wr_max_length} || 1,
4642                                                    $drive->{iops_max_length} || 1,
4643                                                    $drive->{iops_rd_max_length} || 1,
4644                                                    $drive->{iops_wr_max_length} || 1);
4645
4646                     }
4647
4648                     return 1;
4649                 }
4650
4651             } else { # cdrom
4652
4653                 if ($drive->{file} eq 'none') {
4654                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4655                     if (drive_is_cloudinit($old_drive)) {
4656                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4657                     }
4658                 } else {
4659                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4660                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4661                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4662                 }
4663
4664                 return 1;
4665             }
4666         }
4667     }
4668
4669     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4670     # hotplug new disks
4671     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4672     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4673 }
4674
4675 sub vm_start {
4676     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4677         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4678
4679     PVE::QemuConfig->lock_config($vmid, sub {
4680         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4681
4682         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4683
4684         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4685
4686         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4687
4688         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4689             vmconfig_apply_pending($vmid, $conf, $storecfg);
4690             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4691         }
4692
4693         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4694
4695         my $defaults = load_defaults();
4696
4697         # set environment variable useful inside network script
4698         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4699
4700         my $local_volumes = {};
4701
4702         if ($targetstorage) {
4703             foreach_drive($conf, sub {
4704                 my ($ds, $drive) = @_;
4705
4706                 return if drive_is_cdrom($drive);
4707
4708                 my $volid = $drive->{file};
4709
4710                 return if !$volid;
4711
4712                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4713
4714                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4715                 return if $scfg->{shared};
4716                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4717             });
4718
4719             my $format = undef;
4720
4721             foreach my $opt (sort keys %$local_volumes) {
4722
4723                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4724                 my $drive = parse_drive($opt, $conf->{$opt});
4725
4726                 #if remote storage is specified, use default format
4727                 if ($targetstorage && $targetstorage ne "1") {
4728                     $storeid = $targetstorage;
4729                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4730                     $format = $defFormat;
4731                 } else {
4732                     #else we use same format than original
4733                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4734                     $format = qemu_img_format($scfg, $volid);
4735                 }
4736
4737                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4738                 my $newdrive = $drive;
4739                 $newdrive->{format} = $format;
4740                 $newdrive->{file} = $newvolid;
4741                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4742                 $local_volumes->{$opt} = $drivestr;
4743                 #pass drive to conf for command line
4744                 $conf->{$opt} = $drivestr;
4745             }
4746         }
4747
4748         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4749
4750         my $migrate_port = 0;
4751         my $migrate_uri;
4752         if ($statefile) {
4753             if ($statefile eq 'tcp') {
4754                 my $localip = "localhost";
4755                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4756                 my $nodename = PVE::INotify::nodename();
4757
4758                 if (!defined($migration_type)) {
4759                     if (defined($datacenterconf->{migration}->{type})) {
4760                         $migration_type = $datacenterconf->{migration}->{type};
4761                     } else {
4762                         $migration_type = 'secure';
4763                     }
4764                 }
4765
4766                 if ($migration_type eq 'insecure') {
4767                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4768                     if ($migrate_network_addr) {
4769                         $localip = $migrate_network_addr;
4770                     } else {
4771                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4772                     }
4773
4774                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4775                 }
4776
4777                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4778                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4779                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4780                 push @$cmd, '-incoming', $migrate_uri;
4781                 push @$cmd, '-S';
4782
4783             } elsif ($statefile eq 'unix') {
4784                 # should be default for secure migrations as a ssh TCP forward
4785                 # tunnel is not deterministic reliable ready and fails regurarly
4786                 # to set up in time, so use UNIX socket forwards
4787                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4788                 unlink $socket_addr;
4789
4790                 $migrate_uri = "unix:$socket_addr";
4791
4792                 push @$cmd, '-incoming', $migrate_uri;
4793                 push @$cmd, '-S';
4794
4795             } else {
4796                 push @$cmd, '-loadstate', $statefile;
4797             }
4798         } elsif ($paused) {
4799             push @$cmd, '-S';
4800         }
4801
4802         # host pci devices
4803         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4804           my $d = parse_hostpci($conf->{"hostpci$i"});
4805           next if !$d;
4806           my $pcidevices = $d->{pciid};
4807           foreach my $pcidevice (@$pcidevices) {
4808                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4809
4810                 my $info = pci_device_info("0000:$pciid");
4811                 die "IOMMU not present\n" if !check_iommu_support();
4812                 die "no pci device info for device '$pciid'\n" if !$info;
4813                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4814                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4815           }
4816         }
4817
4818         PVE::Storage::activate_volumes($storecfg, $vollist);
4819
4820         if (!check_running($vmid, 1)) {
4821             eval {
4822                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4823                     outfunc => sub {}, errfunc => sub {});
4824             };
4825         }
4826
4827         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4828                                                   : $defaults->{cpuunits};
4829
4830         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4831         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4832
4833         my %properties = (
4834             Slice => 'qemu.slice',
4835             KillMode => 'none',
4836             CPUShares => $cpuunits
4837         );
4838
4839         if (my $cpulimit = $conf->{cpulimit}) {
4840             $properties{CPUQuota} = int($cpulimit * 100);
4841         }
4842         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4843
4844         my $run_qemu = sub {
4845             PVE::Tools::run_fork sub {
4846                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4847                 run_command($cmd, %run_params);
4848             };
4849         };
4850
4851         if ($conf->{hugepages}) {
4852
4853             my $code = sub {
4854                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4855                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4856
4857                 PVE::QemuServer::Memory::hugepages_mount();
4858                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4859
4860                 eval { $run_qemu->() };
4861                 if (my $err = $@) {
4862                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4863                     die $err;
4864                 }
4865
4866                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4867             };
4868             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4869
4870         } else {
4871             eval { $run_qemu->() };
4872         }
4873
4874         if (my $err = $@) {
4875             # deactivate volumes if start fails
4876             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4877             die "start failed: $err";
4878         }
4879
4880         print "migration listens on $migrate_uri\n" if $migrate_uri;
4881
4882         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4883             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4884             warn $@ if $@;
4885         }
4886
4887         #start nbd server for storage migration
4888         if ($targetstorage) {
4889             my $nodename = PVE::INotify::nodename();
4890             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4891             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4892             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4893             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4894
4895             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4896
4897             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4898
4899             foreach my $opt (sort keys %$local_volumes) {
4900                 my $volid = $local_volumes->{$opt};
4901                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4902                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4903                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4904             }
4905         }
4906
4907         if ($migratedfrom) {
4908             eval {
4909                 set_migration_caps($vmid);
4910             };
4911             warn $@ if $@;
4912
4913             if ($spice_port) {
4914                 print "spice listens on port $spice_port\n";
4915                 if ($spice_ticket) {
4916                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4917                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4918                 }
4919             }
4920
4921         } else {
4922             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4923                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4924                     if $conf->{balloon};
4925             }
4926
4927             foreach my $opt (keys %$conf) {
4928                 next if $opt !~  m/^net\d+$/;
4929                 my $nicconf = parse_net($conf->{$opt});
4930                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4931             }
4932         }
4933
4934         vm_mon_cmd_nocheck($vmid, 'qom-set',
4935                     path => "machine/peripheral/balloon0",
4936                     property => "guest-stats-polling-interval",
4937                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4938
4939     });
4940 }
4941
4942 sub vm_mon_cmd {
4943     my ($vmid, $execute, %params) = @_;
4944
4945     my $cmd = { execute => $execute, arguments => \%params };
4946     vm_qmp_command($vmid, $cmd);
4947 }
4948
4949 sub vm_mon_cmd_nocheck {
4950     my ($vmid, $execute, %params) = @_;
4951
4952     my $cmd = { execute => $execute, arguments => \%params };
4953     vm_qmp_command($vmid, $cmd, 1);
4954 }
4955
4956 sub vm_qmp_command {
4957     my ($vmid, $cmd, $nocheck) = @_;
4958
4959     my $res;
4960
4961     my $timeout;
4962     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4963         $timeout = $cmd->{arguments}->{timeout};
4964         delete $cmd->{arguments}->{timeout};
4965     }
4966
4967     eval {
4968         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4969         my $sname = qmp_socket($vmid);
4970         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4971             my $qmpclient = PVE::QMPClient->new();
4972
4973             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4974         } else {
4975             die "unable to open monitor socket\n";
4976         }
4977     };
4978     if (my $err = $@) {
4979         syslog("err", "VM $vmid qmp command failed - $err");
4980         die $err;
4981     }
4982
4983     return $res;
4984 }
4985
4986 sub vm_human_monitor_command {
4987     my ($vmid, $cmdline) = @_;
4988
4989     my $res;
4990
4991     my $cmd = {
4992         execute => 'human-monitor-command',
4993         arguments => { 'command-line' => $cmdline},
4994     };
4995
4996     return vm_qmp_command($vmid, $cmd);
4997 }
4998
4999 sub vm_commandline {
5000     my ($storecfg, $vmid) = @_;
5001
5002     my $conf = PVE::QemuConfig->load_config($vmid);
5003
5004     my $defaults = load_defaults();
5005
5006     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5007
5008     return PVE::Tools::cmd2string($cmd);
5009 }
5010
5011 sub vm_reset {
5012     my ($vmid, $skiplock) = @_;
5013
5014     PVE::QemuConfig->lock_config($vmid, sub {
5015
5016         my $conf = PVE::QemuConfig->load_config($vmid);
5017
5018         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5019
5020         vm_mon_cmd($vmid, "system_reset");
5021     });
5022 }
5023
5024 sub get_vm_volumes {
5025     my ($conf) = @_;
5026
5027     my $vollist = [];
5028     foreach_volid($conf, sub {
5029         my ($volid, $attr) = @_;
5030
5031         return if $volid =~ m|^/|;
5032
5033         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5034         return if !$sid;
5035
5036         push @$vollist, $volid;
5037     });
5038
5039     return $vollist;
5040 }
5041
5042 sub vm_stop_cleanup {
5043     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5044
5045     eval {
5046
5047         if (!$keepActive) {
5048             my $vollist = get_vm_volumes($conf);
5049             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5050         }
5051
5052         foreach my $ext (qw(mon qmp pid vnc qga)) {
5053             unlink "/var/run/qemu-server/${vmid}.$ext";
5054         }
5055
5056         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5057     };
5058     warn $@ if $@; # avoid errors - just warn
5059 }
5060
5061 # Note: use $nockeck to skip tests if VM configuration file exists.
5062 # We need that when migration VMs to other nodes (files already moved)
5063 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5064 sub vm_stop {
5065     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5066
5067     $force = 1 if !defined($force) && !$shutdown;
5068
5069     if ($migratedfrom){
5070         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5071         kill 15, $pid if $pid;
5072         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5073         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5074         return;
5075     }
5076
5077     PVE::QemuConfig->lock_config($vmid, sub {
5078
5079         my $pid = check_running($vmid, $nocheck);
5080         return if !$pid;
5081
5082         my $conf;
5083         if (!$nocheck) {
5084             $conf = PVE::QemuConfig->load_config($vmid);
5085             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5086             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5087                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5088                 $timeout = $opts->{down} if $opts->{down};
5089             }
5090         }
5091
5092         $timeout = 60 if !defined($timeout);
5093
5094         eval {
5095             if ($shutdown) {
5096                 if (defined($conf) && $conf->{agent}) {
5097                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5098                 } else {
5099                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5100                 }
5101             } else {
5102                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5103             }
5104         };
5105         my $err = $@;
5106
5107         if (!$err) {
5108             my $count = 0;
5109             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5110                 $count++;
5111                 sleep 1;
5112             }
5113
5114             if ($count >= $timeout) {
5115                 if ($force) {
5116                     warn "VM still running - terminating now with SIGTERM\n";
5117                     kill 15, $pid;
5118                 } else {
5119                     die "VM quit/powerdown failed - got timeout\n";
5120                 }
5121             } else {
5122                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5123                 return;
5124             }
5125         } else {
5126             if ($force) {
5127                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5128                 kill 15, $pid;
5129             } else {
5130                 die "VM quit/powerdown failed\n";
5131             }
5132         }
5133
5134         # wait again
5135         $timeout = 10;
5136
5137         my $count = 0;
5138         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5139             $count++;
5140             sleep 1;
5141         }
5142
5143         if ($count >= $timeout) {
5144             warn "VM still running - terminating now with SIGKILL\n";
5145             kill 9, $pid;
5146             sleep 1;
5147         }
5148
5149         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5150    });
5151 }
5152
5153 sub vm_suspend {
5154     my ($vmid, $skiplock) = @_;
5155
5156     PVE::QemuConfig->lock_config($vmid, sub {
5157
5158         my $conf = PVE::QemuConfig->load_config($vmid);
5159
5160         PVE::QemuConfig->check_lock($conf)
5161             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5162
5163         vm_mon_cmd($vmid, "stop");
5164     });
5165 }
5166
5167 sub vm_resume {
5168     my ($vmid, $skiplock, $nocheck) = @_;
5169
5170     PVE::QemuConfig->lock_config($vmid, sub {
5171
5172         if (!$nocheck) {
5173
5174             my $conf = PVE::QemuConfig->load_config($vmid);
5175
5176             PVE::QemuConfig->check_lock($conf)
5177                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5178
5179             vm_mon_cmd($vmid, "cont");
5180
5181         } else {
5182             vm_mon_cmd_nocheck($vmid, "cont");
5183         }
5184     });
5185 }
5186
5187 sub vm_sendkey {
5188     my ($vmid, $skiplock, $key) = @_;
5189
5190     PVE::QemuConfig->lock_config($vmid, sub {
5191
5192         my $conf = PVE::QemuConfig->load_config($vmid);
5193
5194         # there is no qmp command, so we use the human monitor command
5195         vm_human_monitor_command($vmid, "sendkey $key");
5196     });
5197 }
5198
5199 sub vm_destroy {
5200     my ($storecfg, $vmid, $skiplock) = @_;
5201
5202     PVE::QemuConfig->lock_config($vmid, sub {
5203
5204         my $conf = PVE::QemuConfig->load_config($vmid);
5205
5206         if (!check_running($vmid)) {
5207             destroy_vm($storecfg, $vmid, undef, $skiplock);
5208         } else {
5209             die "VM $vmid is running - destroy failed\n";
5210         }
5211     });
5212 }
5213
5214 # pci helpers
5215
5216 sub file_write {
5217     my ($filename, $buf) = @_;
5218
5219     my $fh = IO::File->new($filename, "w");
5220     return undef if !$fh;
5221
5222     my $res = print $fh $buf;
5223
5224     $fh->close();
5225
5226     return $res;
5227 }
5228
5229 sub pci_device_info {
5230     my ($name) = @_;
5231
5232     my $res;
5233
5234     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5235     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5236
5237     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5238     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5239
5240     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5241     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5242
5243     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5244     return undef if !defined($product) || $product !~ s/^0x//;
5245
5246     $res = {
5247         name => $name,
5248         vendor => $vendor,
5249         product => $product,
5250         domain => $domain,
5251         bus => $bus,
5252         slot => $slot,
5253         func => $func,
5254         irq => $irq,
5255         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5256     };
5257
5258     return $res;
5259 }
5260
5261 sub pci_dev_reset {
5262     my ($dev) = @_;
5263
5264     my $name = $dev->{name};
5265
5266     my $fn = "$pcisysfs/devices/$name/reset";
5267
5268     return file_write($fn, "1");
5269 }
5270
5271 sub pci_dev_bind_to_vfio {
5272     my ($dev) = @_;
5273
5274     my $name = $dev->{name};
5275
5276     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5277
5278     if (!-d $vfio_basedir) {
5279         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5280     }
5281     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5282
5283     my $testdir = "$vfio_basedir/$name";
5284     return 1 if -d $testdir;
5285
5286     my $data = "$dev->{vendor} $dev->{product}";
5287     return undef if !file_write("$vfio_basedir/new_id", $data);
5288
5289     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5290     if (!file_write($fn, $name)) {
5291         return undef if -f $fn;
5292     }
5293
5294     $fn = "$vfio_basedir/bind";
5295     if (! -d $testdir) {
5296         return undef if !file_write($fn, $name);
5297     }
5298
5299     return -d $testdir;
5300 }
5301
5302 sub pci_dev_group_bind_to_vfio {
5303     my ($pciid) = @_;
5304
5305     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5306
5307     if (!-d $vfio_basedir) {
5308         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5309     }
5310     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5311
5312     # get IOMMU group devices
5313     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5314       my @devs = grep /^0000:/, readdir($D);
5315     closedir($D);
5316
5317     foreach my $pciid (@devs) {
5318         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5319
5320         # pci bridges, switches or root ports are not supported
5321         # they have a pci_bus subdirectory so skip them
5322         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5323
5324         my $info = pci_device_info($1);
5325         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5326     }
5327
5328     return 1;
5329 }
5330
5331 # vzdump restore implementaion
5332
5333 sub tar_archive_read_firstfile {
5334     my $archive = shift;
5335
5336     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5337
5338     # try to detect archive type first
5339     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5340         die "unable to open file '$archive'\n";
5341     my $firstfile = <$fh>;
5342     kill 15, $pid;
5343     close $fh;
5344
5345     die "ERROR: archive contaions no data\n" if !$firstfile;
5346     chomp $firstfile;
5347
5348     return $firstfile;
5349 }
5350
5351 sub tar_restore_cleanup {
5352     my ($storecfg, $statfile) = @_;
5353
5354     print STDERR "starting cleanup\n";
5355
5356     if (my $fd = IO::File->new($statfile, "r")) {
5357         while (defined(my $line = <$fd>)) {
5358             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5359                 my $volid = $2;
5360                 eval {
5361                     if ($volid =~ m|^/|) {
5362                         unlink $volid || die 'unlink failed\n';
5363                     } else {
5364                         PVE::Storage::vdisk_free($storecfg, $volid);
5365                     }
5366                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5367                 };
5368                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5369             } else {
5370                 print STDERR "unable to parse line in statfile - $line";
5371             }
5372         }
5373         $fd->close();
5374     }
5375 }
5376
5377 sub restore_archive {
5378     my ($archive, $vmid, $user, $opts) = @_;
5379
5380     my $format = $opts->{format};
5381     my $comp;
5382
5383     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5384         $format = 'tar' if !$format;
5385         $comp = 'gzip';
5386     } elsif ($archive =~ m/\.tar$/) {
5387         $format = 'tar' if !$format;
5388     } elsif ($archive =~ m/.tar.lzo$/) {
5389         $format = 'tar' if !$format;
5390         $comp = 'lzop';
5391     } elsif ($archive =~ m/\.vma$/) {
5392         $format = 'vma' if !$format;
5393     } elsif ($archive =~ m/\.vma\.gz$/) {
5394         $format = 'vma' if !$format;
5395         $comp = 'gzip';
5396     } elsif ($archive =~ m/\.vma\.lzo$/) {
5397         $format = 'vma' if !$format;
5398         $comp = 'lzop';
5399     } else {
5400         $format = 'vma' if !$format; # default
5401     }
5402
5403     # try to detect archive format
5404     if ($format eq 'tar') {
5405         return restore_tar_archive($archive, $vmid, $user, $opts);
5406     } else {
5407         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5408     }
5409 }
5410
5411 sub restore_update_config_line {
5412     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5413
5414     return if $line =~ m/^\#qmdump\#/;
5415     return if $line =~ m/^\#vzdump\#/;
5416     return if $line =~ m/^lock:/;
5417     return if $line =~ m/^unused\d+:/;
5418     return if $line =~ m/^parent:/;
5419     return if $line =~ m/^template:/; # restored VM is never a template
5420
5421     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5422     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5423         # try to convert old 1.X settings
5424         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5425         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5426             my ($model, $macaddr) = split(/\=/, $devconfig);
5427             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5428             my $net = {
5429                 model => $model,
5430                 bridge => "vmbr$ind",
5431                 macaddr => $macaddr,
5432             };
5433             my $netstr = print_net($net);
5434
5435             print $outfd "net$cookie->{netcount}: $netstr\n";
5436             $cookie->{netcount}++;
5437         }
5438     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5439         my ($id, $netstr) = ($1, $2);
5440         my $net = parse_net($netstr);
5441         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5442         $netstr = print_net($net);
5443         print $outfd "$id: $netstr\n";
5444     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5445         my $virtdev = $1;
5446         my $value = $3;
5447         my $di = parse_drive($virtdev, $value);
5448         if (defined($di->{backup}) && !$di->{backup}) {
5449             print $outfd "#$line";
5450         } elsif ($map->{$virtdev}) {
5451             delete $di->{format}; # format can change on restore
5452             $di->{file} = $map->{$virtdev};
5453             $value = print_drive($vmid, $di);
5454             print $outfd "$virtdev: $value\n";
5455         } else {
5456             print $outfd $line;
5457         }
5458     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5459         my ($uuid, $uuid_str);
5460         UUID::generate($uuid);
5461         UUID::unparse($uuid, $uuid_str);
5462         my $smbios1 = parse_smbios1($2);
5463         $smbios1->{uuid} = $uuid_str;
5464         print $outfd $1.print_smbios1($smbios1)."\n";
5465     } else {
5466         print $outfd $line;
5467     }
5468 }
5469
5470 sub scan_volids {
5471     my ($cfg, $vmid) = @_;
5472
5473     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5474
5475     my $volid_hash = {};
5476     foreach my $storeid (keys %$info) {
5477         foreach my $item (@{$info->{$storeid}}) {
5478             next if !($item->{volid} && $item->{size});
5479             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5480             $volid_hash->{$item->{volid}} = $item;
5481         }
5482     }
5483
5484     return $volid_hash;
5485 }
5486
5487 sub is_volume_in_use {
5488     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5489
5490     my $path = PVE::Storage::path($storecfg, $volid);
5491
5492     my $scan_config = sub {
5493         my ($cref, $snapname) = @_;
5494
5495         foreach my $key (keys %$cref) {
5496             my $value = $cref->{$key};
5497             if (is_valid_drivename($key)) {
5498                 next if $skip_drive && $key eq $skip_drive;
5499                 my $drive = parse_drive($key, $value);
5500                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5501                 return 1 if $volid eq $drive->{file};
5502                 if ($drive->{file} =~ m!^/!) {
5503                     return 1 if $drive->{file} eq $path;
5504                 } else {
5505                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5506                     next if !$storeid;
5507                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5508                     next if !$scfg;
5509                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5510                 }
5511             }
5512         }
5513
5514         return 0;
5515     };
5516
5517     return 1 if &$scan_config($conf);
5518
5519     undef $skip_drive;
5520
5521     foreach my $snapname (keys %{$conf->{snapshots}}) {
5522         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5523     }
5524
5525     return 0;
5526 }
5527
5528 sub update_disksize {
5529     my ($vmid, $conf, $volid_hash) = @_;
5530
5531     my $changes;
5532
5533     # used and unused disks
5534     my $referenced = {};
5535
5536     # Note: it is allowed to define multiple storages with same path (alias), so
5537     # we need to check both 'volid' and real 'path' (two different volid can point
5538     # to the same path).
5539
5540     my $referencedpath = {};
5541
5542     # update size info
5543     foreach my $opt (keys %$conf) {
5544         if (is_valid_drivename($opt)) {
5545             my $drive = parse_drive($opt, $conf->{$opt});
5546             my $volid = $drive->{file};
5547             next if !$volid;
5548
5549             $referenced->{$volid} = 1;
5550             if ($volid_hash->{$volid} &&
5551                 (my $path = $volid_hash->{$volid}->{path})) {
5552                 $referencedpath->{$path} = 1;
5553             }
5554
5555             next if drive_is_cdrom($drive);
5556             next if !$volid_hash->{$volid};
5557
5558             $drive->{size} = $volid_hash->{$volid}->{size};
5559             my $new = print_drive($vmid, $drive);
5560             if ($new ne $conf->{$opt}) {
5561                 $changes = 1;
5562                 $conf->{$opt} = $new;
5563             }
5564         }
5565     }
5566
5567     # remove 'unusedX' entry if volume is used
5568     foreach my $opt (keys %$conf) {
5569         next if $opt !~ m/^unused\d+$/;
5570         my $volid = $conf->{$opt};
5571         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5572         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5573             $changes = 1;
5574             delete $conf->{$opt};
5575         }
5576
5577         $referenced->{$volid} = 1;
5578         $referencedpath->{$path} = 1 if $path;
5579     }
5580
5581     foreach my $volid (sort keys %$volid_hash) {
5582         next if $volid =~ m/vm-$vmid-state-/;
5583         next if $referenced->{$volid};
5584         my $path = $volid_hash->{$volid}->{path};
5585         next if !$path; # just to be sure
5586         next if $referencedpath->{$path};
5587         $changes = 1;
5588         PVE::QemuConfig->add_unused_volume($conf, $volid);
5589         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5590     }
5591
5592     return $changes;
5593 }
5594
5595 sub rescan {
5596     my ($vmid, $nolock) = @_;
5597
5598     my $cfg = PVE::Storage::config();
5599
5600     my $volid_hash = scan_volids($cfg, $vmid);
5601
5602     my $updatefn =  sub {
5603         my ($vmid) = @_;
5604
5605         my $conf = PVE::QemuConfig->load_config($vmid);
5606
5607         PVE::QemuConfig->check_lock($conf);
5608
5609         my $vm_volids = {};
5610         foreach my $volid (keys %$volid_hash) {
5611             my $info = $volid_hash->{$volid};
5612             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5613         }
5614
5615         my $changes = update_disksize($vmid, $conf, $vm_volids);
5616
5617         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5618     };
5619
5620     if (defined($vmid)) {
5621         if ($nolock) {
5622             &$updatefn($vmid);
5623         } else {
5624             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5625         }
5626     } else {
5627         my $vmlist = config_list();
5628         foreach my $vmid (keys %$vmlist) {
5629             if ($nolock) {
5630                 &$updatefn($vmid);
5631             } else {
5632                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5633             }
5634         }
5635     }
5636 }
5637
5638 sub restore_vma_archive {
5639     my ($archive, $vmid, $user, $opts, $comp) = @_;
5640
5641     my $readfrom = $archive;
5642
5643     my $cfg = PVE::Storage::config();
5644     my $commands = [];
5645     my $bwlimit = $opts->{bwlimit};
5646
5647     my $dbg_cmdstring = '';
5648     my $add_pipe = sub {
5649         my ($cmd) = @_;
5650         push @$commands, $cmd;
5651         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5652         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5653         $readfrom = '-';
5654     };
5655
5656     my $input = undef;
5657     if ($archive eq '-') {
5658         $input = '<&STDIN';
5659     } else {
5660         # If we use a backup from a PVE defined storage we also consider that
5661         # storage's rate limit:
5662         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5663         if (defined($volid)) {
5664             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5665             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5666             if ($readlimit) {
5667                 print STDERR "applying read rate limit: $readlimit\n";
5668                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5669                 $add_pipe->($cstream);
5670             }
5671         }
5672     }
5673
5674     if ($comp) {
5675         my $cmd;
5676         if ($comp eq 'gzip') {
5677             $cmd = ['zcat', $readfrom];
5678         } elsif ($comp eq 'lzop') {
5679             $cmd = ['lzop', '-d', '-c', $readfrom];
5680         } else {
5681             die "unknown compression method '$comp'\n";
5682         }
5683         $add_pipe->($cmd);
5684     }
5685
5686     my $tmpdir = "/var/tmp/vzdumptmp$$";
5687     rmtree $tmpdir;
5688
5689     # disable interrupts (always do cleanups)
5690     local $SIG{INT} =
5691         local $SIG{TERM} =
5692         local $SIG{QUIT} =
5693         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5694
5695     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5696     POSIX::mkfifo($mapfifo, 0600);
5697     my $fifofh;
5698
5699     my $openfifo = sub {
5700         open($fifofh, '>', $mapfifo) || die $!;
5701     };
5702
5703     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5704
5705     my $oldtimeout;
5706     my $timeout = 5;
5707
5708     my $devinfo = {};
5709
5710     my $rpcenv = PVE::RPCEnvironment::get();
5711
5712     my $conffile = PVE::QemuConfig->config_file($vmid);
5713     my $tmpfn = "$conffile.$$.tmp";
5714
5715     # Note: $oldconf is undef if VM does not exists
5716     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5717     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5718
5719     my %storage_limits;
5720
5721     my $print_devmap = sub {
5722         my $virtdev_hash = {};
5723
5724         my $cfgfn = "$tmpdir/qemu-server.conf";
5725
5726         # we can read the config - that is already extracted
5727         my $fh = IO::File->new($cfgfn, "r") ||
5728             "unable to read qemu-server.conf - $!\n";
5729
5730         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5731         if (-f $fwcfgfn) {
5732             my $pve_firewall_dir = '/etc/pve/firewall';
5733             mkdir $pve_firewall_dir; # make sure the dir exists
5734             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5735         }
5736
5737         while (defined(my $line = <$fh>)) {
5738             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5739                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5740                 die "archive does not contain data for drive '$virtdev'\n"
5741                     if !$devinfo->{$devname};
5742                 if (defined($opts->{storage})) {
5743                     $storeid = $opts->{storage} || 'local';
5744                 } elsif (!$storeid) {
5745                     $storeid = 'local';
5746                 }
5747                 $format = 'raw' if !$format;
5748                 $devinfo->{$devname}->{devname} = $devname;
5749                 $devinfo->{$devname}->{virtdev} = $virtdev;
5750                 $devinfo->{$devname}->{format} = $format;
5751                 $devinfo->{$devname}->{storeid} = $storeid;
5752
5753                 # check permission on storage
5754                 my $pool = $opts->{pool}; # todo: do we need that?
5755                 if ($user ne 'root@pam') {
5756                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5757                 }
5758
5759                 $storage_limits{$storeid} = $bwlimit;
5760
5761                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5762             }
5763         }
5764
5765         foreach my $key (keys %storage_limits) {
5766             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5767             next if !$limit;
5768             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5769             $storage_limits{$key} = $limit * 1024;
5770         }
5771
5772         foreach my $devname (keys %$devinfo) {
5773             die "found no device mapping information for device '$devname'\n"
5774                 if !$devinfo->{$devname}->{virtdev};
5775         }
5776
5777         # create empty/temp config
5778         if ($oldconf) {
5779             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5780             foreach_drive($oldconf, sub {
5781                 my ($ds, $drive) = @_;
5782
5783                 return if drive_is_cdrom($drive);
5784
5785                 my $volid = $drive->{file};
5786
5787                 return if !$volid || $volid =~ m|^/|;
5788
5789                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5790                 return if !$path || !$owner || ($owner != $vmid);
5791
5792                 # Note: only delete disk we want to restore
5793                 # other volumes will become unused
5794                 if ($virtdev_hash->{$ds}) {
5795                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5796                     if (my $err = $@) {
5797                         warn $err;
5798                     }
5799                 }
5800             });
5801
5802             # delete vmstate files
5803             # since after the restore we have no snapshots anymore
5804             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5805                 my $snap = $oldconf->{snapshots}->{$snapname};
5806                 if ($snap->{vmstate}) {
5807                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5808                     if (my $err = $@) {
5809                         warn $err;
5810                     }
5811                 }
5812             }
5813         }
5814
5815         my $map = {};
5816         foreach my $virtdev (sort keys %$virtdev_hash) {
5817             my $d = $virtdev_hash->{$virtdev};
5818             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5819             my $storeid = $d->{storeid};
5820             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5821
5822             my $map_opts = '';
5823             if (my $limit = $storage_limits{$storeid}) {
5824                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5825             }
5826
5827             # test if requested format is supported
5828             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5829             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5830             $d->{format} = $defFormat if !$supported;
5831
5832             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5833                                                   $d->{format}, undef, $alloc_size);
5834             print STDERR "new volume ID is '$volid'\n";
5835             $d->{volid} = $volid;
5836             my $path = PVE::Storage::path($cfg, $volid);
5837
5838             PVE::Storage::activate_volumes($cfg,[$volid]);
5839
5840             my $write_zeros = 1;
5841             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5842                 $write_zeros = 0;
5843             }
5844
5845             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5846
5847             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5848             $map->{$virtdev} = $volid;
5849         }
5850
5851         $fh->seek(0, 0) || die "seek failed - $!\n";
5852
5853         my $outfd = new IO::File ($tmpfn, "w") ||
5854             die "unable to write config for VM $vmid\n";
5855
5856         my $cookie = { netcount => 0 };
5857         while (defined(my $line = <$fh>)) {
5858             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5859         }
5860
5861         $fh->close();
5862         $outfd->close();
5863     };
5864
5865     eval {
5866         # enable interrupts
5867         local $SIG{INT} =
5868             local $SIG{TERM} =
5869             local $SIG{QUIT} =
5870             local $SIG{HUP} =
5871             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5872         local $SIG{ALRM} = sub { die "got timeout\n"; };
5873
5874         $oldtimeout = alarm($timeout);
5875
5876         my $parser = sub {
5877             my $line = shift;
5878
5879             print "$line\n";
5880
5881             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5882                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5883                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5884             } elsif ($line =~ m/^CTIME: /) {
5885                 # we correctly received the vma config, so we can disable
5886                 # the timeout now for disk allocation (set to 10 minutes, so
5887                 # that we always timeout if something goes wrong)
5888                 alarm(600);
5889                 &$print_devmap();
5890                 print $fifofh "done\n";
5891                 my $tmp = $oldtimeout || 0;
5892                 $oldtimeout = undef;
5893                 alarm($tmp);
5894                 close($fifofh);
5895             }
5896         };
5897
5898         print "restore vma archive: $dbg_cmdstring\n";
5899         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5900     };
5901     my $err = $@;
5902
5903     alarm($oldtimeout) if $oldtimeout;
5904
5905     my $vollist = [];
5906     foreach my $devname (keys %$devinfo) {
5907         my $volid = $devinfo->{$devname}->{volid};
5908         push @$vollist, $volid if $volid;
5909     }
5910
5911     PVE::Storage::deactivate_volumes($cfg, $vollist);
5912
5913     unlink $mapfifo;
5914
5915     if ($err) {
5916         rmtree $tmpdir;
5917         unlink $tmpfn;
5918
5919         foreach my $devname (keys %$devinfo) {
5920             my $volid = $devinfo->{$devname}->{volid};
5921             next if !$volid;
5922             eval {
5923                 if ($volid =~ m|^/|) {
5924                     unlink $volid || die 'unlink failed\n';
5925                 } else {
5926                     PVE::Storage::vdisk_free($cfg, $volid);
5927                 }
5928                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5929             };
5930             print STDERR "unable to cleanup '$volid' - $@" if $@;
5931         }
5932         die $err;
5933     }
5934
5935     rmtree $tmpdir;
5936
5937     rename($tmpfn, $conffile) ||
5938         die "unable to commit configuration file '$conffile'\n";
5939
5940     PVE::Cluster::cfs_update(); # make sure we read new file
5941
5942     eval { rescan($vmid, 1); };
5943     warn $@ if $@;
5944 }
5945
5946 sub restore_tar_archive {
5947     my ($archive, $vmid, $user, $opts) = @_;
5948
5949     if ($archive ne '-') {
5950         my $firstfile = tar_archive_read_firstfile($archive);
5951         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5952             if $firstfile ne 'qemu-server.conf';
5953     }
5954
5955     my $storecfg = PVE::Storage::config();
5956
5957     # destroy existing data - keep empty config
5958     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5959     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5960
5961     my $tocmd = "/usr/lib/qemu-server/qmextract";
5962
5963     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5964     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5965     $tocmd .= ' --prealloc' if $opts->{prealloc};
5966     $tocmd .= ' --info' if $opts->{info};
5967
5968     # tar option "xf" does not autodetect compression when read from STDIN,
5969     # so we pipe to zcat
5970     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5971         PVE::Tools::shellquote("--to-command=$tocmd");
5972
5973     my $tmpdir = "/var/tmp/vzdumptmp$$";
5974     mkpath $tmpdir;
5975
5976     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5977     local $ENV{VZDUMP_VMID} = $vmid;
5978     local $ENV{VZDUMP_USER} = $user;
5979
5980     my $conffile = PVE::QemuConfig->config_file($vmid);
5981     my $tmpfn = "$conffile.$$.tmp";
5982
5983     # disable interrupts (always do cleanups)
5984     local $SIG{INT} =
5985         local $SIG{TERM} =
5986         local $SIG{QUIT} =
5987         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5988
5989     eval {
5990         # enable interrupts
5991         local $SIG{INT} =
5992             local $SIG{TERM} =
5993             local $SIG{QUIT} =
5994             local $SIG{HUP} =
5995             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5996
5997         if ($archive eq '-') {
5998             print "extracting archive from STDIN\n";
5999             run_command($cmd, input => "<&STDIN");
6000         } else {
6001             print "extracting archive '$archive'\n";
6002             run_command($cmd);
6003         }
6004
6005         return if $opts->{info};
6006
6007         # read new mapping
6008         my $map = {};
6009         my $statfile = "$tmpdir/qmrestore.stat";
6010         if (my $fd = IO::File->new($statfile, "r")) {
6011             while (defined (my $line = <$fd>)) {
6012                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6013                     $map->{$1} = $2 if $1;
6014                 } else {
6015                     print STDERR "unable to parse line in statfile - $line\n";
6016                 }
6017             }
6018             $fd->close();
6019         }
6020
6021         my $confsrc = "$tmpdir/qemu-server.conf";
6022
6023         my $srcfd = new IO::File($confsrc, "r") ||
6024             die "unable to open file '$confsrc'\n";
6025
6026         my $outfd = new IO::File ($tmpfn, "w") ||
6027             die "unable to write config for VM $vmid\n";
6028
6029         my $cookie = { netcount => 0 };
6030         while (defined (my $line = <$srcfd>)) {
6031             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6032         }
6033
6034         $srcfd->close();
6035         $outfd->close();
6036     };
6037     my $err = $@;
6038
6039     if ($err) {
6040
6041         unlink $tmpfn;
6042
6043         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6044
6045         die $err;
6046     }
6047
6048     rmtree $tmpdir;
6049
6050     rename $tmpfn, $conffile ||
6051         die "unable to commit configuration file '$conffile'\n";
6052
6053     PVE::Cluster::cfs_update(); # make sure we read new file
6054
6055     eval { rescan($vmid, 1); };
6056     warn $@ if $@;
6057 };
6058
6059 sub foreach_storage_used_by_vm {
6060     my ($conf, $func) = @_;
6061
6062     my $sidhash = {};
6063
6064     foreach_drive($conf, sub {
6065         my ($ds, $drive) = @_;
6066         return if drive_is_cdrom($drive);
6067
6068         my $volid = $drive->{file};
6069
6070         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6071         $sidhash->{$sid} = $sid if $sid;
6072     });
6073
6074     foreach my $sid (sort keys %$sidhash) {
6075         &$func($sid);
6076     }
6077 }
6078
6079 sub do_snapshots_with_qemu {
6080     my ($storecfg, $volid) = @_;
6081
6082     my $storage_name = PVE::Storage::parse_volume_id($volid);
6083
6084     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6085         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6086         return 1;
6087     }
6088
6089     if ($volid =~ m/\.(qcow2|qed)$/){
6090         return 1;
6091     }
6092
6093     return undef;
6094 }
6095
6096 sub qga_check_running {
6097     my ($vmid) = @_;
6098
6099     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6100     if ($@) {
6101         warn "Qemu Guest Agent is not running - $@";
6102         return 0;
6103     }
6104     return 1;
6105 }
6106
6107 sub template_create {
6108     my ($vmid, $conf, $disk) = @_;
6109
6110     my $storecfg = PVE::Storage::config();
6111
6112     foreach_drive($conf, sub {
6113         my ($ds, $drive) = @_;
6114
6115         return if drive_is_cdrom($drive);
6116         return if $disk && $ds ne $disk;
6117
6118         my $volid = $drive->{file};
6119         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6120
6121         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6122         $drive->{file} = $voliddst;
6123         $conf->{$ds} = print_drive($vmid, $drive);
6124         PVE::QemuConfig->write_config($vmid, $conf);
6125     });
6126 }
6127
6128 sub qemu_img_convert {
6129     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6130
6131     my $storecfg = PVE::Storage::config();
6132     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6133     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6134
6135     if ($src_storeid && $dst_storeid) {
6136
6137         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6138
6139         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6140         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6141
6142         my $src_format = qemu_img_format($src_scfg, $src_volname);
6143         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6144
6145         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6146         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6147
6148         my $cmd = [];
6149         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6150         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6151         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6152         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6153         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6154         if ($is_zero_initialized) {
6155             push @$cmd, "zeroinit:$dst_path";
6156         } else {
6157             push @$cmd, $dst_path;
6158         }
6159
6160         my $parser = sub {
6161             my $line = shift;
6162             if($line =~ m/\((\S+)\/100\%\)/){
6163                 my $percent = $1;
6164                 my $transferred = int($size * $percent / 100);
6165                 my $remaining = $size - $transferred;
6166
6167                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6168             }
6169
6170         };
6171
6172         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6173         my $err = $@;
6174         die "copy failed: $err" if $err;
6175     }
6176 }
6177
6178 sub qemu_img_format {
6179     my ($scfg, $volname) = @_;
6180
6181     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6182         return $1;
6183     } else {
6184         return "raw";
6185     }
6186 }
6187
6188 sub qemu_drive_mirror {
6189     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6190
6191     $jobs = {} if !$jobs;
6192
6193     my $qemu_target;
6194     my $format;
6195     $jobs->{"drive-$drive"} = {};
6196
6197     if ($dst_volid =~ /^nbd:/) {
6198         $qemu_target = $dst_volid;
6199         $format = "nbd";
6200     } else {
6201         my $storecfg = PVE::Storage::config();
6202         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6203
6204         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6205
6206         $format = qemu_img_format($dst_scfg, $dst_volname);
6207
6208         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6209
6210         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6211     }
6212
6213     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6214     $opts->{format} = $format if $format;
6215
6216     print "drive mirror is starting for drive-$drive\n";
6217
6218     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6219
6220     if (my $err = $@) {
6221         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6222         die "mirroring error: $err";
6223     }
6224
6225     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6226 }
6227
6228 sub qemu_drive_mirror_monitor {
6229     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6230
6231     eval {
6232         my $err_complete = 0;
6233
6234         while (1) {
6235             die "storage migration timed out\n" if $err_complete > 300;
6236
6237             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6238
6239             my $running_mirror_jobs = {};
6240             foreach my $stat (@$stats) {
6241                 next if $stat->{type} ne 'mirror';
6242                 $running_mirror_jobs->{$stat->{device}} = $stat;
6243             }
6244
6245             my $readycounter = 0;
6246
6247             foreach my $job (keys %$jobs) {
6248
6249                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6250                     print "$job : finished\n";
6251                     delete $jobs->{$job};
6252                     next;
6253                 }
6254
6255                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6256
6257                 my $busy = $running_mirror_jobs->{$job}->{busy};
6258                 my $ready = $running_mirror_jobs->{$job}->{ready};
6259                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6260                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6261                     my $remaining = $total - $transferred;
6262                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6263
6264                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6265                 }
6266
6267                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6268             }
6269
6270             last if scalar(keys %$jobs) == 0;
6271
6272             if ($readycounter == scalar(keys %$jobs)) {
6273                 print "all mirroring jobs are ready \n";
6274                 last if $skipcomplete; #do the complete later
6275
6276                 if ($vmiddst && $vmiddst != $vmid) {
6277                     my $agent_running = $qga && qga_check_running($vmid);
6278                     if ($agent_running) {
6279                         print "freeze filesystem\n";
6280                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6281                     } else {
6282                         print "suspend vm\n";
6283                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6284                     }
6285
6286                     # if we clone a disk for a new target vm, we don't switch the disk
6287                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6288
6289                     if ($agent_running) {
6290                         print "unfreeze filesystem\n";
6291                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6292                     } else {
6293                         print "resume vm\n";
6294                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6295                     }
6296
6297                     last;
6298                 } else {
6299
6300                     foreach my $job (keys %$jobs) {
6301                         # try to switch the disk if source and destination are on the same guest
6302                         print "$job: Completing block job...\n";
6303
6304                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6305                         if ($@ =~ m/cannot be completed/) {
6306                             print "$job: Block job cannot be completed, try again.\n";
6307                             $err_complete++;
6308                         }else {
6309                             print "$job: Completed successfully.\n";
6310                             $jobs->{$job}->{complete} = 1;
6311                         }
6312                     }
6313                 }
6314             }
6315             sleep 1;
6316         }
6317     };
6318     my $err = $@;
6319
6320     if ($err) {
6321         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6322         die "mirroring error: $err";
6323     }
6324
6325 }
6326
6327 sub qemu_blockjobs_cancel {
6328     my ($vmid, $jobs) = @_;
6329
6330     foreach my $job (keys %$jobs) {
6331         print "$job: Cancelling block job\n";
6332         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6333         $jobs->{$job}->{cancel} = 1;
6334     }
6335
6336     while (1) {
6337         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6338
6339         my $running_jobs = {};
6340         foreach my $stat (@$stats) {
6341             $running_jobs->{$stat->{device}} = $stat;
6342         }
6343
6344         foreach my $job (keys %$jobs) {
6345
6346             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6347                 print "$job: Done.\n";
6348                 delete $jobs->{$job};
6349             }
6350         }
6351
6352         last if scalar(keys %$jobs) == 0;
6353
6354         sleep 1;
6355     }
6356 }
6357
6358 sub clone_disk {
6359     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6360         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6361
6362     my $newvolid;
6363
6364     if (!$full) {
6365         print "create linked clone of drive $drivename ($drive->{file})\n";
6366         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6367         push @$newvollist, $newvolid;
6368     } else {
6369
6370         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6371         $storeid = $storage if $storage;
6372
6373         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6374         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6375
6376         print "create full clone of drive $drivename ($drive->{file})\n";
6377         my $name = undef;
6378         if (drive_is_cloudinit($drive)) {
6379             $name = "vm-$newvmid-cloudinit";
6380             # cloudinit only supports raw and qcow2 atm:
6381             if ($dst_format eq 'qcow2') {
6382                 $name .= '.qcow2';
6383             } elsif ($dst_format ne 'raw') {
6384                 die "clone: unhandled format for cloudinit image\n";
6385             }
6386         }
6387         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6388         push @$newvollist, $newvolid;
6389
6390         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6391
6392         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6393         if (!$running || $snapname) {
6394             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6395         } else {
6396
6397             my $kvmver = get_running_qemu_version ($vmid);
6398             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6399                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6400                     if $drive->{iothread};
6401             }
6402
6403             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6404         }
6405     }
6406
6407     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6408
6409     my $disk = $drive;
6410     $disk->{format} = undef;
6411     $disk->{file} = $newvolid;
6412     $disk->{size} = $size;
6413
6414     return $disk;
6415 }
6416
6417 # this only works if VM is running
6418 sub get_current_qemu_machine {
6419     my ($vmid) = @_;
6420
6421     my $cmd = { execute => 'query-machines', arguments => {} };
6422     my $res = vm_qmp_command($vmid, $cmd);
6423
6424     my ($current, $default);
6425     foreach my $e (@$res) {
6426         $default = $e->{name} if $e->{'is-default'};
6427         $current = $e->{name} if $e->{'is-current'};
6428     }
6429
6430     # fallback to the default machine if current is not supported by qemu
6431     return $current || $default || 'pc';
6432 }
6433
6434 sub get_running_qemu_version {
6435     my ($vmid) = @_;
6436     my $cmd = { execute => 'query-version', arguments => {} };
6437     my $res = vm_qmp_command($vmid, $cmd);
6438     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6439 }
6440
6441 sub qemu_machine_feature_enabled {
6442     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6443
6444     my $current_major;
6445     my $current_minor;
6446
6447     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6448
6449         $current_major = $3;
6450         $current_minor = $4;
6451
6452     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6453
6454         $current_major = $1;
6455         $current_minor = $2;
6456     }
6457
6458     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6459
6460
6461 }
6462
6463 sub qemu_machine_pxe {
6464     my ($vmid, $conf, $machine) = @_;
6465
6466     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6467
6468     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6469         $machine .= '.pxe';
6470     }
6471
6472     return $machine;
6473 }
6474
6475 sub qemu_use_old_bios_files {
6476     my ($machine_type) = @_;
6477
6478     return if !$machine_type;
6479
6480     my $use_old_bios_files = undef;
6481
6482     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6483         $machine_type = $1;
6484         $use_old_bios_files = 1;
6485     } else {
6486         my $kvmver = kvm_user_version();
6487         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6488         # load new efi bios files on migration. So this hack is required to allow
6489         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6490         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6491         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6492     }
6493
6494     return ($use_old_bios_files, $machine_type);
6495 }
6496
6497 sub create_efidisk {
6498     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6499
6500     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6501
6502     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6503     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6504     PVE::Storage::activate_volumes($storecfg, [$volid]);
6505
6506     my $path = PVE::Storage::path($storecfg, $volid);
6507     eval {
6508         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6509     };
6510     die "Copying EFI vars image failed: $@" if $@;
6511
6512     return ($volid, $vars_size);
6513 }
6514
6515 sub lspci {
6516
6517     my $devices = {};
6518
6519     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6520             my (undef, $id, $function) = @_;
6521             my $res = { id => $id, function => $function};
6522             push @{$devices->{$id}}, $res;
6523     });
6524
6525     # Entries should be sorted by functions.
6526     foreach my $id (keys %$devices) {
6527         my $dev = $devices->{$id};
6528         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6529     }
6530
6531     return $devices;
6532 }
6533
6534 sub vm_iothreads_list {
6535     my ($vmid) = @_;
6536
6537     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6538
6539     my $iothreads = {};
6540     foreach my $iothread (@$res) {
6541         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6542     }
6543
6544     return $iothreads;
6545 }
6546
6547 sub scsihw_infos {
6548     my ($conf, $drive) = @_;
6549
6550     my $maxdev = 0;
6551
6552     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6553         $maxdev = 7;
6554     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6555         $maxdev = 1;
6556     } else {
6557         $maxdev = 256;
6558     }
6559
6560     my $controller = int($drive->{index} / $maxdev);
6561     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6562
6563     return ($maxdev, $controller, $controller_prefix);
6564 }
6565
6566 sub add_hyperv_enlightenments {
6567     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6568
6569     return if $winversion < 6;
6570     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6571
6572     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6573
6574     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6575         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6576         push @$cpuFlags , 'hv_vapic';
6577         push @$cpuFlags , 'hv_time';
6578     } else {
6579         push @$cpuFlags , 'hv_spinlocks=0xffff';
6580     }
6581
6582     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6583         push @$cpuFlags , 'hv_reset';
6584         push @$cpuFlags , 'hv_vpindex';
6585         push @$cpuFlags , 'hv_runtime';
6586     }
6587
6588     if ($winversion >= 7) {
6589         push @$cpuFlags , 'hv_relaxed';
6590     }
6591 }
6592
6593 sub windows_version {
6594     my ($ostype) = @_;
6595
6596     return 0 if !$ostype;
6597
6598     my $winversion = 0;
6599
6600     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6601         $winversion = 5;
6602     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6603         $winversion = 6;
6604     } elsif ($ostype =~ m/^win(\d+)$/) {
6605         $winversion = $1;
6606     }
6607
6608     return $winversion;
6609 }
6610
6611 sub resolve_dst_disk_format {
6612         my ($storecfg, $storeid, $src_volname, $format) = @_;
6613         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6614
6615         if (!$format) {
6616             # if no target format is specified, use the source disk format as hint
6617             if ($src_volname) {
6618                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6619                 $format = qemu_img_format($scfg, $src_volname);
6620             } else {
6621                 return $defFormat;
6622             }
6623         }
6624
6625         # test if requested format is supported - else use default
6626         my $supported = grep { $_ eq $format } @$validFormats;
6627         $format = $defFormat if !$supported;
6628         return $format;
6629 }
6630
6631 sub resolve_first_disk {
6632     my $conf = shift;
6633     my @disks = PVE::QemuServer::valid_drive_names();
6634     my $firstdisk;
6635     foreach my $ds (reverse @disks) {
6636         next if !$conf->{$ds};
6637         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6638         next if PVE::QemuServer::drive_is_cdrom($disk);
6639         $firstdisk = $ds;
6640     }
6641     return $firstdisk;
6642 }
6643
6644 sub generate_smbios1_uuid {
6645     my ($uuid, $uuid_str);
6646     UUID::generate($uuid);
6647     UUID::unparse($uuid, $uuid_str);
6648     return "uuid=$uuid_str";
6649 }
6650
6651 # bash completion helper
6652
6653 sub complete_backup_archives {
6654     my ($cmdname, $pname, $cvalue) = @_;
6655
6656     my $cfg = PVE::Storage::config();
6657
6658     my $storeid;
6659
6660     if ($cvalue =~ m/^([^:]+):/) {
6661         $storeid = $1;
6662     }
6663
6664     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6665
6666     my $res = [];
6667     foreach my $id (keys %$data) {
6668         foreach my $item (@{$data->{$id}}) {
6669             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6670             push @$res, $item->{volid} if defined($item->{volid});
6671         }
6672     }
6673
6674     return $res;
6675 }
6676
6677 my $complete_vmid_full = sub {
6678     my ($running) = @_;
6679
6680     my $idlist = vmstatus();
6681
6682     my $res = [];
6683
6684     foreach my $id (keys %$idlist) {
6685         my $d = $idlist->{$id};
6686         if (defined($running)) {
6687             next if $d->{template};
6688             next if $running && $d->{status} ne 'running';
6689             next if !$running && $d->{status} eq 'running';
6690         }
6691         push @$res, $id;
6692
6693     }
6694     return $res;
6695 };
6696
6697 sub complete_vmid {
6698     return &$complete_vmid_full();
6699 }
6700
6701 sub complete_vmid_stopped {
6702     return &$complete_vmid_full(0);
6703 }
6704
6705 sub complete_vmid_running {
6706     return &$complete_vmid_full(1);
6707 }
6708
6709 sub complete_storage {
6710
6711     my $cfg = PVE::Storage::config();
6712     my $ids = $cfg->{ids};
6713
6714     my $res = [];
6715     foreach my $sid (keys %$ids) {
6716         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6717         next if !$ids->{$sid}->{content}->{images};
6718         push @$res, $sid;
6719     }
6720
6721     return $res;
6722 }
6723
6724 sub nbd_stop {
6725     my ($vmid) = @_;
6726
6727     vm_mon_cmd($vmid, 'nbd-server-stop');
6728 }
6729
6730 1;