PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $confdesc = {
 260     onboot => {
 261         optional => 1,
 262         type => 'boolean',
 263         description => "Specifies whether a VM will be started during system bootup.",
 264         default => 0,
 265     },
 266     autostart => {
 267         optional => 1,
 268         type => 'boolean',
 269         description => "Automatic restart after crash (currently ignored).",
 270         default => 0,
 271     },
 272     hotplug => {
 273         optional => 1,
 274         type => 'string', format => 'pve-hotplug-features',
 275         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 276         default => 'network,disk,usb',
 277     },
 278     reboot => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 282         default => 1,
 283     },
 284     lock => {
 285         optional => 1,
 286         type => 'string',
 287         description => "Lock/unlock the VM.",
 288         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 289     },
 290     cpulimit => {
 291         optional => 1,
 292         type => 'number',
 293         description => "Limit of CPU usage.",
 294         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 295         minimum => 0,
 296         maximum => 128,
 297         default => 0,
 298     },
 299     cpuunits => {
 300         optional => 1,
 301         type => 'integer',
 302         description => "CPU weight for a VM.",
 303         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 304         minimum => 2,
 305         maximum => 262144,
 306         default => 1024,
 307     },
 308     memory => {
 309         optional => 1,
 310         type => 'integer',
 311         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 312         minimum => 16,
 313         default => 512,
 314     },
 315     balloon => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 319         minimum => 0,
 320     },
 321     shares => {
 322         optional => 1,
 323         type => 'integer',
 324         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 325         minimum => 0,
 326         maximum => 50000,
 327         default => 1000,
 328     },
 329     keyboard => {
 330         optional => 1,
 331         type => 'string',
 332         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 333                        "It should not be necessary to set it.",
 334         enum => PVE::Tools::kvmkeymaplist(),
 335         default => undef,
 336     },
 337     name => {
 338         optional => 1,
 339         type => 'string', format => 'dns-name',
 340         description => "Set a name for the VM. Only used on the configuration web interface.",
 341     },
 342     scsihw => {
 343         optional => 1,
 344         type => 'string',
 345         description => "SCSI controller model",
 346         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 347         default => 'lsi',
 348     },
 349     description => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 353     },
 354     ostype => {
 355         optional => 1,
 356         type => 'string',
 357         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 358         description => "Specify guest operating system.",
 359         verbose_description => <<EODESC,
 360 Specify guest operating system. This is used to enable special
 361 optimization/features for specific operating systems:
 362
 363 [horizontal]
 364 other;; unspecified OS
 365 wxp;; Microsoft Windows XP
 366 w2k;; Microsoft Windows 2000
 367 w2k3;; Microsoft Windows 2003
 368 w2k8;; Microsoft Windows 2008
 369 wvista;; Microsoft Windows Vista
 370 win7;; Microsoft Windows 7
 371 win8;; Microsoft Windows 8/2012/2012r2
 372 win10;; Microsoft Windows 10/2016
 373 l24;; Linux 2.4 Kernel
 374 l26;; Linux 2.6/3.X Kernel
 375 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 376 EODESC
 377     },
 378     boot => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 382         pattern => '[acdn]{1,4}',
 383         default => 'cdn',
 384     },
 385     bootdisk => {
 386         optional => 1,
 387         type => 'string', format => 'pve-qm-bootdisk',
 388         description => "Enable booting from specified disk.",
 389         pattern => '(ide|sata|scsi|virtio)\d+',
 390     },
 391     smp => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "The number of CPUs. Please use option -sockets instead.",
 395         minimum => 1,
 396         default => 1,
 397     },
 398     sockets => {
 399         optional => 1,
 400         type => 'integer',
 401         description => "The number of CPU sockets.",
 402         minimum => 1,
 403         default => 1,
 404     },
 405     cores => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of cores per socket.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     numa => {
 413         optional => 1,
 414         type => 'boolean',
 415         description => "Enable/disable NUMA.",
 416         default => 0,
 417     },
 418     hugepages => {
 419         optional => 1,
 420         type => 'string',
 421         description => "Enable/disable hugepages memory.",
 422         enum => [qw(any 2 1024)],
 423     },
 424     vcpus => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "Number of hotplugged vcpus.",
 428         minimum => 1,
 429         default => 0,
 430     },
 431     acpi => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable ACPI.",
 435         default => 1,
 436     },
 437     agent => {
 438         optional => 1,
 439         description => "Enable/disable Qemu GuestAgent and its properties.",
 440         type => 'string',
 441         format => $agent_fmt,
 442     },
 443     kvm => {
 444         optional => 1,
 445         type => 'boolean',
 446         description => "Enable/disable KVM hardware virtualization.",
 447         default => 1,
 448     },
 449     tdf => {
 450         optional => 1,
 451         type => 'boolean',
 452         description => "Enable/disable time drift fix.",
 453         default => 0,
 454     },
 455     localtime => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 459     },
 460     freeze => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 464     },
 465     vga => {
 466         optional => 1,
 467         type => 'string', format => $vga_fmt,
 468         description => "Configure the VGA hardware.",
 469         verbose_description => "Configure the VGA Hardware. If you want to use ".
 470             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 471             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 472             "is 'std' for all OS types besides some Windows versions (XP and " .
 473             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 474             "display server. For win* OS you can select how many independent " .
 475             "displays you want, Linux guests can add displays them self.\n".
 476             "You can also run without any graphic card, using a serial device as terminal.",
 477     },
 478     watchdog => {
 479         optional => 1,
 480         type => 'string', format => 'pve-qm-watchdog',
 481         description => "Create a virtual hardware watchdog device.",
 482         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 483             " (by a guest action), the watchdog must be periodically polled " .
 484             "by an agent inside the guest or else the watchdog will reset " .
 485             "the guest (or execute the respective action specified)",
 486     },
 487     startdate => {
 488         optional => 1,
 489         type => 'string',
 490         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 491         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 492         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 493         default => 'now',
 494     },
 495     startup =>  get_standard_option('pve-startup-order'),
 496     template => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable Template.",
 500         default => 0,
 501     },
 502     args => {
 503         optional => 1,
 504         type => 'string',
 505         description => "Arbitrary arguments passed to kvm.",
 506         verbose_description => <<EODESCR,
 507 Arbitrary arguments passed to kvm, for example:
 508
 509 args: -no-reboot -no-hpet
 510
 511 NOTE: this option is for experts only.
 512 EODESCR
 513     },
 514     tablet => {
 515         optional => 1,
 516         type => 'boolean',
 517         default => 1,
 518         description => "Enable/disable the USB tablet device.",
 519         verbose_description => "Enable/disable the USB tablet device. This device is " .
 520             "usually needed to allow absolute mouse positioning with VNC. " .
 521             "Else the mouse runs out of sync with normal VNC clients. " .
 522             "If you're running lots of console-only guests on one host, " .
 523             "you may consider disabling this to save some context switches. " .
 524             "This is turned off by default if you use spice (-vga=qxl).",
 525     },
 526     migrate_speed => {
 527         optional => 1,
 528         type => 'integer',
 529         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 530         minimum => 0,
 531         default => 0,
 532     },
 533     migrate_downtime => {
 534         optional => 1,
 535         type => 'number',
 536         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 537         minimum => 0,
 538         default => 0.1,
 539     },
 540     cdrom => {
 541         optional => 1,
 542         type => 'string', format => 'pve-qm-ide',
 543         typetext => '<volume>',
 544         description => "This is an alias for option -ide2",
 545     },
 546     cpu => {
 547         optional => 1,
 548         description => "Emulated CPU type.",
 549         type => 'string',
 550         format => $cpu_fmt,
 551     },
 552     parent => get_standard_option('pve-snapshot-name', {
 553         optional => 1,
 554         description => "Parent snapshot name. This is used internally, and should not be modified.",
 555     }),
 556     snaptime => {
 557         optional => 1,
 558         description => "Timestamp for snapshots.",
 559         type => 'integer',
 560         minimum => 0,
 561     },
 562     vmstate => {
 563         optional => 1,
 564         type => 'string', format => 'pve-volume-id',
 565         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 566     },
 567     vmstatestorage => get_standard_option('pve-storage-id', {
 568         description => "Default storage for VM state volumes/files.",
 569         optional => 1,
 570     }),
 571     runningmachine => get_standard_option('pve-qemu-machine', {
 572         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 573     }),
 574     machine => get_standard_option('pve-qemu-machine'),
 575     arch => {
 576         description => "Virtual processor architecture. Defaults to the host.",
 577         optional => 1,
 578         type => 'string',
 579         enum => [qw(x86_64 aarch64)],
 580     },
 581     smbios1 => {
 582         description => "Specify SMBIOS type 1 fields.",
 583         type => 'string', format => 'pve-qm-smbios1',
 584         maxLength => 256,
 585         optional => 1,
 586     },
 587     protection => {
 588         optional => 1,
 589         type => 'boolean',
 590         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 591         default => 0,
 592     },
 593     bios => {
 594         optional => 1,
 595         type => 'string',
 596         enum => [ qw(seabios ovmf) ],
 597         description => "Select BIOS implementation.",
 598         default => 'seabios',
 599     },
 600     vmgenid => {
 601         type => 'string',
 602         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 603         format_description => 'UUID',
 604         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 605         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 606             " 128-bit integer value identifier to the guest OS. This allows to".
 607             " notify the guest operating system when the virtual machine is".
 608             " executed with a different configuration (e.g. snapshot execution".
 609             " or creation from a template). The guest operating system notices".
 610             " the change, and is then able to react as appropriate by marking".
 611             " its copies of distributed databases as dirty, re-initializing its".
 612             " random number generator, etc.\n".
 613             "Note that auto-creation only works when done throug API/CLI create".
 614             " or update methods, but not when manually editing the config file.",
 615         default => "1 (autogenerated)",
 616         optional => 1,
 617     },
 618     hookscript => {
 619         type => 'string',
 620         format => 'pve-volume-id',
 621         optional => 1,
 622         description => "Script that will be executed during various steps in the vms lifetime.",
 623     },
 624 };
 625
 626 my $confdesc_cloudinit = {
 627     citype => {
 628         optional => 1,
 629         type => 'string',
 630         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 631         enum => ['configdrive2', 'nocloud'],
 632     },
 633     ciuser => {
 634         optional => 1,
 635         type => 'string',
 636         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 637     },
 638     cipassword => {
 639         optional => 1,
 640         type => 'string',
 641         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 642     },
 643     searchdomain => {
 644         optional => 1,
 645         type => 'string',
 646         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 647     },
 648     nameserver => {
 649         optional => 1,
 650         type => 'string', format => 'address-list',
 651         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 652     },
 653     sshkeys => {
 654         optional => 1,
 655         type => 'string',
 656         format => 'urlencoded',
 657         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 658     },
 659 };
 660
 661 # what about other qemu settings ?
 662 #cpu => 'string',
 663 #machine => 'string',
 664 #fda => 'file',
 665 #fdb => 'file',
 666 #mtdblock => 'file',
 667 #sd => 'file',
 668 #pflash => 'file',
 669 #snapshot => 'bool',
 670 #bootp => 'file',
 671 ##tftp => 'dir',
 672 ##smb => 'dir',
 673 #kernel => 'file',
 674 #append => 'string',
 675 #initrd => 'file',
 676 ##soundhw => 'string',
 677
 678 while (my ($k, $v) = each %$confdesc) {
 679     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 680 }
 681
 682 my $MAX_IDE_DISKS = 4;
 683 my $MAX_SCSI_DISKS = 14;
 684 my $MAX_VIRTIO_DISKS = 16;
 685 my $MAX_SATA_DISKS = 6;
 686 my $MAX_USB_DEVICES = 5;
 687 my $MAX_NETS = 32;
 688 my $MAX_UNUSED_DISKS = 256;
 689 my $MAX_HOSTPCI_DEVICES = 4;
 690 my $MAX_SERIAL_PORTS = 4;
 691 my $MAX_PARALLEL_PORTS = 3;
 692 my $MAX_NUMA = 8;
 693
 694 my $numa_fmt = {
 695     cpus => {
 696         type => "string",
 697         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 698         description => "CPUs accessing this NUMA node.",
 699         format_description => "id[-id];...",
 700     },
 701     memory => {
 702         type => "number",
 703         description => "Amount of memory this NUMA node provides.",
 704         optional => 1,
 705     },
 706     hostnodes => {
 707         type => "string",
 708         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 709         description => "Host NUMA nodes to use.",
 710         format_description => "id[-id];...",
 711         optional => 1,
 712     },
 713     policy => {
 714         type => 'string',
 715         enum => [qw(preferred bind interleave)],
 716         description => "NUMA allocation policy.",
 717         optional => 1,
 718     },
 719 };
 720 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 721 my $numadesc = {
 722     optional => 1,
 723     type => 'string', format => $numa_fmt,
 724     description => "NUMA topology.",
 725 };
 726 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 727
 728 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 729     $confdesc->{"numa$i"} = $numadesc;
 730 }
 731
 732 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 733                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 734                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 735 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 736
 737 my $net_fmt_bridge_descr = <<__EOD__;
 738 Bridge to attach the network device to. The Proxmox VE standard bridge
 739 is called 'vmbr0'.
 740
 741 If you do not specify a bridge, we create a kvm user (NATed) network
 742 device, which provides DHCP and DNS services. The following addresses
 743 are used:
 744
 745  10.0.2.2   Gateway
 746  10.0.2.3   DNS Server
 747  10.0.2.4   SMB Server
 748
 749 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 750 __EOD__
 751
 752 my $net_fmt = {
 753     macaddr => {
 754         type => 'string',
 755         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 756         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 757         format_description => "XX:XX:XX:XX:XX:XX",
 758         optional => 1,
 759     },
 760     model => {
 761         type => 'string',
 762         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 763         enum => $nic_model_list,
 764         default_key => 1,
 765     },
 766     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 767     bridge => {
 768         type => 'string',
 769         description => $net_fmt_bridge_descr,
 770         format_description => 'bridge',
 771         optional => 1,
 772     },
 773     queues => {
 774         type => 'integer',
 775         minimum => 0, maximum => 16,
 776         description => 'Number of packet queues to be used on the device.',
 777         optional => 1,
 778     },
 779     rate => {
 780         type => 'number',
 781         minimum => 0,
 782         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 783         optional => 1,
 784     },
 785     tag => {
 786         type => 'integer',
 787         minimum => 1, maximum => 4094,
 788         description => 'VLAN tag to apply to packets on this interface.',
 789         optional => 1,
 790     },
 791     trunks => {
 792         type => 'string',
 793         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 794         description => 'VLAN trunks to pass through this interface.',
 795         format_description => 'vlanid[;vlanid...]',
 796         optional => 1,
 797     },
 798     firewall => {
 799         type => 'boolean',
 800         description => 'Whether this interface should be protected by the firewall.',
 801         optional => 1,
 802     },
 803     link_down => {
 804         type => 'boolean',
 805         description => 'Whether this interface should be disconnected (like pulling the plug).',
 806         optional => 1,
 807     },
 808 };
 809
 810 my $netdesc = {
 811     optional => 1,
 812     type => 'string', format => $net_fmt,
 813     description => "Specify network devices.",
 814 };
 815
 816 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 817
 818 my $ipconfig_fmt = {
 819     ip => {
 820         type => 'string',
 821         format => 'pve-ipv4-config',
 822         format_description => 'IPv4Format/CIDR',
 823         description => 'IPv4 address in CIDR format.',
 824         optional => 1,
 825         default => 'dhcp',
 826     },
 827     gw => {
 828         type => 'string',
 829         format => 'ipv4',
 830         format_description => 'GatewayIPv4',
 831         description => 'Default gateway for IPv4 traffic.',
 832         optional => 1,
 833         requires => 'ip',
 834     },
 835     ip6 => {
 836         type => 'string',
 837         format => 'pve-ipv6-config',
 838         format_description => 'IPv6Format/CIDR',
 839         description => 'IPv6 address in CIDR format.',
 840         optional => 1,
 841         default => 'dhcp',
 842     },
 843     gw6 => {
 844         type => 'string',
 845         format => 'ipv6',
 846         format_description => 'GatewayIPv6',
 847         description => 'Default gateway for IPv6 traffic.',
 848         optional => 1,
 849         requires => 'ip6',
 850     },
 851 };
 852 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 853 my $ipconfigdesc = {
 854     optional => 1,
 855     type => 'string', format => 'pve-qm-ipconfig',
 856     description => <<'EODESCR',
 857 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 858
 859 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 860
 861 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 862 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 863
 864 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 865 EODESCR
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 868
 869 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 870     $confdesc->{"net$i"} = $netdesc;
 871     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 872 }
 873
 874 foreach my $key (keys %$confdesc_cloudinit) {
 875     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 876 }
 877
 878 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 879 sub verify_volume_id_or_qm_path {
 880     my ($volid, $noerr) = @_;
 881
 882     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 883         return $volid;
 884     }
 885
 886     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 887     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 888     if ($@) {
 889         return undef if $noerr;
 890         die $@;
 891     }
 892     return $volid;
 893 }
 894
 895 my $drivename_hash;
 896
 897 my %drivedesc_base = (
 898     volume => { alias => 'file' },
 899     file => {
 900         type => 'string',
 901         format => 'pve-volume-id-or-qm-path',
 902         default_key => 1,
 903         format_description => 'volume',
 904         description => "The drive's backing volume.",
 905     },
 906     media => {
 907         type => 'string',
 908         enum => [qw(cdrom disk)],
 909         description => "The drive's media type.",
 910         default => 'disk',
 911         optional => 1
 912     },
 913     cyls => {
 914         type => 'integer',
 915         description => "Force the drive's physical geometry to have a specific cylinder count.",
 916         optional => 1
 917     },
 918     heads => {
 919         type => 'integer',
 920         description => "Force the drive's physical geometry to have a specific head count.",
 921         optional => 1
 922     },
 923     secs => {
 924         type => 'integer',
 925         description => "Force the drive's physical geometry to have a specific sector count.",
 926         optional => 1
 927     },
 928     trans => {
 929         type => 'string',
 930         enum => [qw(none lba auto)],
 931         description => "Force disk geometry bios translation mode.",
 932         optional => 1,
 933     },
 934     snapshot => {
 935         type => 'boolean',
 936         description => "Controls qemu's snapshot mode feature."
 937             . " If activated, changes made to the disk are temporary and will"
 938             . " be discarded when the VM is shutdown.",
 939         optional => 1,
 940     },
 941     cache => {
 942         type => 'string',
 943         enum => [qw(none writethrough writeback unsafe directsync)],
 944         description => "The drive's cache mode",
 945         optional => 1,
 946     },
 947     format => get_standard_option('pve-qm-image-format'),
 948     size => {
 949         type => 'string',
 950         format => 'disk-size',
 951         format_description => 'DiskSize',
 952         description => "Disk size. This is purely informational and has no effect.",
 953         optional => 1,
 954     },
 955     backup => {
 956         type => 'boolean',
 957         description => "Whether the drive should be included when making backups.",
 958         optional => 1,
 959     },
 960     replicate => {
 961         type => 'boolean',
 962         description => 'Whether the drive should considered for replication jobs.',
 963         optional => 1,
 964         default => 1,
 965     },
 966     rerror => {
 967         type => 'string',
 968         enum => [qw(ignore report stop)],
 969         description => 'Read error action.',
 970         optional => 1,
 971     },
 972     werror => {
 973         type => 'string',
 974         enum => [qw(enospc ignore report stop)],
 975         description => 'Write error action.',
 976         optional => 1,
 977     },
 978     aio => {
 979         type => 'string',
 980         enum => [qw(native threads)],
 981         description => 'AIO type to use.',
 982         optional => 1,
 983     },
 984     discard => {
 985         type => 'string',
 986         enum => [qw(ignore on)],
 987         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 988         optional => 1,
 989     },
 990     detect_zeroes => {
 991         type => 'boolean',
 992         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 993         optional => 1,
 994     },
 995     serial => {
 996         type => 'string',
 997         format => 'urlencoded',
 998         format_description => 'serial',
 999         maxLength => 20*3, # *3 since it's %xx url enoded
1000         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1001         optional => 1,
1002     },
1003     shared => {
1004         type => 'boolean',
1005         description => 'Mark this locally-managed volume as available on all nodes',
1006         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1007         optional => 1,
1008         default => 0,
1009     }
1010 );
1011
1012 my %iothread_fmt = ( iothread => {
1013         type => 'boolean',
1014         description => "Whether to use iothreads for this drive",
1015         optional => 1,
1016 });
1017
1018 my %model_fmt = (
1019     model => {
1020         type => 'string',
1021         format => 'urlencoded',
1022         format_description => 'model',
1023         maxLength => 40*3, # *3 since it's %xx url enoded
1024         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1025         optional => 1,
1026     },
1027 );
1028
1029 my %queues_fmt = (
1030     queues => {
1031         type => 'integer',
1032         description => "Number of queues.",
1033         minimum => 2,
1034         optional => 1
1035     }
1036 );
1037
1038 my %scsiblock_fmt = (
1039     scsiblock => {
1040         type => 'boolean',
1041         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1042         optional => 1,
1043         default => 0,
1044     },
1045 );
1046
1047 my %ssd_fmt = (
1048     ssd => {
1049         type => 'boolean',
1050         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1051         optional => 1,
1052     },
1053 );
1054
1055 my %wwn_fmt = (
1056     wwn => {
1057         type => 'string',
1058         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1059         format_description => 'wwn',
1060         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1061         optional => 1,
1062     },
1063 );
1064
1065 my $add_throttle_desc = sub {
1066     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1067     my $d = {
1068         type => $type,
1069         format_description => $unit,
1070         description => "Maximum $what in $longunit.",
1071         optional => 1,
1072     };
1073     $d->{minimum} = $minimum if defined($minimum);
1074     $drivedesc_base{$key} = $d;
1075 };
1076 # throughput: (leaky bucket)
1077 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1078 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1079 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1080 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1081 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1082 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1083 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1084 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1085 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1086
1087 # pools: (pool of IO before throttling starts taking effect)
1088 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1089 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1090 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1091 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1092 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1093 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1094
1095 # burst lengths
1096 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1097 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1098 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1099 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1100 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1101 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1102
1103 # legacy support
1104 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1105 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1106 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1107 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1108
1109 my $ide_fmt = {
1110     %drivedesc_base,
1111     %model_fmt,
1112     %ssd_fmt,
1113     %wwn_fmt,
1114 };
1115 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1116
1117 my $idedesc = {
1118     optional => 1,
1119     type => 'string', format => $ide_fmt,
1120     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1123
1124 my $scsi_fmt = {
1125     %drivedesc_base,
1126     %iothread_fmt,
1127     %queues_fmt,
1128     %scsiblock_fmt,
1129     %ssd_fmt,
1130     %wwn_fmt,
1131 };
1132 my $scsidesc = {
1133     optional => 1,
1134     type => 'string', format => $scsi_fmt,
1135     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1136 };
1137 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1138
1139 my $sata_fmt = {
1140     %drivedesc_base,
1141     %ssd_fmt,
1142     %wwn_fmt,
1143 };
1144 my $satadesc = {
1145     optional => 1,
1146     type => 'string', format => $sata_fmt,
1147     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1148 };
1149 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1150
1151 my $virtio_fmt = {
1152     %drivedesc_base,
1153     %iothread_fmt,
1154 };
1155 my $virtiodesc = {
1156     optional => 1,
1157     type => 'string', format => $virtio_fmt,
1158     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1159 };
1160 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1161
1162 my $alldrive_fmt = {
1163     %drivedesc_base,
1164     %iothread_fmt,
1165     %model_fmt,
1166     %queues_fmt,
1167     %scsiblock_fmt,
1168     %ssd_fmt,
1169     %wwn_fmt,
1170 };
1171
1172 my $efidisk_fmt = {
1173     volume => { alias => 'file' },
1174     file => {
1175         type => 'string',
1176         format => 'pve-volume-id-or-qm-path',
1177         default_key => 1,
1178         format_description => 'volume',
1179         description => "The drive's backing volume.",
1180     },
1181     format => get_standard_option('pve-qm-image-format'),
1182     size => {
1183         type => 'string',
1184         format => 'disk-size',
1185         format_description => 'DiskSize',
1186         description => "Disk size. This is purely informational and has no effect.",
1187         optional => 1,
1188     },
1189 };
1190
1191 my $efidisk_desc = {
1192     optional => 1,
1193     type => 'string', format => $efidisk_fmt,
1194     description => "Configure a Disk for storing EFI vars",
1195 };
1196
1197 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1198
1199 my $usb_fmt = {
1200     host => {
1201         default_key => 1,
1202         type => 'string', format => 'pve-qm-usb-device',
1203         format_description => 'HOSTUSBDEVICE|spice',
1204         description => <<EODESCR,
1205 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1206
1207  'bus-port(.port)*' (decimal numbers) or
1208  'vendor_id:product_id' (hexadeciaml numbers) or
1209  'spice'
1210
1211 You can use the 'lsusb -t' command to list existing usb devices.
1212
1213 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1214
1215 The value 'spice' can be used to add a usb redirection devices for spice.
1216 EODESCR
1217     },
1218     usb3 => {
1219         optional => 1,
1220         type => 'boolean',
1221         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1222         default => 0,
1223     },
1224 };
1225
1226 my $usbdesc = {
1227     optional => 1,
1228     type => 'string', format => $usb_fmt,
1229     description => "Configure an USB device (n is 0 to 4).",
1230 };
1231 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1232
1233 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1234 my $hostpci_fmt = {
1235     host => {
1236         default_key => 1,
1237         type => 'string',
1238         pattern => qr/$PCIRE(;$PCIRE)*/,
1239         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1240         description => <<EODESCR,
1241 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1242 of PCI virtual functions of the host. HOSTPCIID syntax is:
1243
1244 'bus:dev.func' (hexadecimal numbers)
1245
1246 You can us the 'lspci' command to list existing PCI devices.
1247 EODESCR
1248     },
1249     rombar => {
1250         type => 'boolean',
1251         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1252         optional => 1,
1253         default => 1,
1254     },
1255     romfile => {
1256         type => 'string',
1257         pattern => '[^,;]+',
1258         format_description => 'string',
1259         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1260         optional => 1,
1261     },
1262     pcie => {
1263         type => 'boolean',
1264         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1265         optional => 1,
1266         default => 0,
1267     },
1268     'x-vga' => {
1269         type => 'boolean',
1270         description =>  "Enable vfio-vga device support.",
1271         optional => 1,
1272         default => 0,
1273     },
1274     'mdev' => {
1275         type => 'string',
1276         format_description => 'string',
1277         pattern => '[^/\.:]+',
1278         optional => 1,
1279         description => <<EODESCR
1280 The type of mediated device to use.
1281 An instance of this type will be created on startup of the VM and
1282 will be cleaned up when the VM stops.
1283 EODESCR
1284     }
1285 };
1286 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1287
1288 my $hostpcidesc = {
1289         optional => 1,
1290         type => 'string', format => 'pve-qm-hostpci',
1291         description => "Map host PCI devices into guest.",
1292         verbose_description =>  <<EODESCR,
1293 Map host PCI devices into guest.
1294
1295 NOTE: This option allows direct access to host hardware. So it is no longer
1296 possible to migrate such machines - use with special care.
1297
1298 CAUTION: Experimental! User reported problems with this option.
1299 EODESCR
1300 };
1301 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1302
1303 my $serialdesc = {
1304         optional => 1,
1305         type => 'string',
1306         pattern => '(/dev/.+|socket)',
1307         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1308         verbose_description =>  <<EODESCR,
1309 Create a serial device inside the VM (n is 0 to 3), and pass through a
1310 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1311 host side (use 'qm terminal' to open a terminal connection).
1312
1313 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1314
1315 CAUTION: Experimental! User reported problems with this option.
1316 EODESCR
1317 };
1318
1319 my $paralleldesc= {
1320         optional => 1,
1321         type => 'string',
1322         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1323         description =>  "Map host parallel devices (n is 0 to 2).",
1324         verbose_description =>  <<EODESCR,
1325 Map host parallel devices (n is 0 to 2).
1326
1327 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1328
1329 CAUTION: Experimental! User reported problems with this option.
1330 EODESCR
1331 };
1332
1333 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1334     $confdesc->{"parallel$i"} = $paralleldesc;
1335 }
1336
1337 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1338     $confdesc->{"serial$i"} = $serialdesc;
1339 }
1340
1341 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1342     $confdesc->{"hostpci$i"} = $hostpcidesc;
1343 }
1344
1345 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1346     $drivename_hash->{"ide$i"} = 1;
1347     $confdesc->{"ide$i"} = $idedesc;
1348 }
1349
1350 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1351     $drivename_hash->{"sata$i"} = 1;
1352     $confdesc->{"sata$i"} = $satadesc;
1353 }
1354
1355 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1356     $drivename_hash->{"scsi$i"} = 1;
1357     $confdesc->{"scsi$i"} = $scsidesc ;
1358 }
1359
1360 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1361     $drivename_hash->{"virtio$i"} = 1;
1362     $confdesc->{"virtio$i"} = $virtiodesc;
1363 }
1364
1365 $drivename_hash->{efidisk0} = 1;
1366 $confdesc->{efidisk0} = $efidisk_desc;
1367
1368 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1369     $confdesc->{"usb$i"} = $usbdesc;
1370 }
1371
1372 my $unuseddesc = {
1373     optional => 1,
1374     type => 'string', format => 'pve-volume-id',
1375     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1376 };
1377
1378 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1379     $confdesc->{"unused$i"} = $unuseddesc;
1380 }
1381
1382 my $kvm_api_version = 0;
1383
1384 sub kvm_version {
1385     return $kvm_api_version if $kvm_api_version;
1386
1387     open my $fh, '<', '/dev/kvm'
1388         or return undef;
1389
1390     # 0xae00 => KVM_GET_API_VERSION
1391     $kvm_api_version = ioctl($fh, 0xae00, 0);
1392
1393     return $kvm_api_version;
1394 }
1395
1396 my $kvm_user_version;
1397
1398 sub kvm_user_version {
1399
1400     return $kvm_user_version if $kvm_user_version;
1401
1402     $kvm_user_version = 'unknown';
1403
1404     my $code = sub {
1405         my $line = shift;
1406         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1407             $kvm_user_version = $2;
1408         }
1409     };
1410
1411     eval { run_command("kvm -version", outfunc => $code); };
1412     warn $@ if $@;
1413
1414     return $kvm_user_version;
1415
1416 }
1417
1418 sub kernel_has_vhost_net {
1419     return -c '/dev/vhost-net';
1420 }
1421
1422 sub valid_drive_names {
1423     # order is important - used to autoselect boot disk
1424     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1425             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1426             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1427             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1428             'efidisk0');
1429 }
1430
1431 sub is_valid_drivename {
1432     my $dev = shift;
1433
1434     return defined($drivename_hash->{$dev});
1435 }
1436
1437 sub option_exists {
1438     my $key = shift;
1439     return defined($confdesc->{$key});
1440 }
1441
1442 sub nic_models {
1443     return $nic_model_list;
1444 }
1445
1446 sub os_list_description {
1447
1448     return {
1449         other => 'Other',
1450         wxp => 'Windows XP',
1451         w2k => 'Windows 2000',
1452         w2k3 =>, 'Windows 2003',
1453         w2k8 => 'Windows 2008',
1454         wvista => 'Windows Vista',
1455         win7 => 'Windows 7',
1456         win8 => 'Windows 8/2012',
1457         win10 => 'Windows 10/2016',
1458         l24 => 'Linux 2.4',
1459         l26 => 'Linux 2.6',
1460     };
1461 }
1462
1463 my $cdrom_path;
1464
1465 sub get_cdrom_path {
1466
1467     return  $cdrom_path if $cdrom_path;
1468
1469     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1470     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1471     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1472 }
1473
1474 sub get_iso_path {
1475     my ($storecfg, $vmid, $cdrom) = @_;
1476
1477     if ($cdrom eq 'cdrom') {
1478         return get_cdrom_path();
1479     } elsif ($cdrom eq 'none') {
1480         return '';
1481     } elsif ($cdrom =~ m|^/|) {
1482         return $cdrom;
1483     } else {
1484         return PVE::Storage::path($storecfg, $cdrom);
1485     }
1486 }
1487
1488 # try to convert old style file names to volume IDs
1489 sub filename_to_volume_id {
1490     my ($vmid, $file, $media) = @_;
1491
1492      if (!($file eq 'none' || $file eq 'cdrom' ||
1493           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1494
1495         return undef if $file =~ m|/|;
1496
1497         if ($media && $media eq 'cdrom') {
1498             $file = "local:iso/$file";
1499         } else {
1500             $file = "local:$vmid/$file";
1501         }
1502     }
1503
1504     return $file;
1505 }
1506
1507 sub verify_media_type {
1508     my ($opt, $vtype, $media) = @_;
1509
1510     return if !$media;
1511
1512     my $etype;
1513     if ($media eq 'disk') {
1514         $etype = 'images';
1515     } elsif ($media eq 'cdrom') {
1516         $etype = 'iso';
1517     } else {
1518         die "internal error";
1519     }
1520
1521     return if ($vtype eq $etype);
1522
1523     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1524 }
1525
1526 sub cleanup_drive_path {
1527     my ($opt, $storecfg, $drive) = @_;
1528
1529     # try to convert filesystem paths to volume IDs
1530
1531     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1532         ($drive->{file} !~ m|^/dev/.+|) &&
1533         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1534         ($drive->{file} !~ m/^\d+$/)) {
1535         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1536         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1537         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1538         verify_media_type($opt, $vtype, $drive->{media});
1539         $drive->{file} = $volid;
1540     }
1541
1542     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1543 }
1544
1545 sub parse_hotplug_features {
1546     my ($data) = @_;
1547
1548     my $res = {};
1549
1550     return $res if $data eq '0';
1551
1552     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1553
1554     foreach my $feature (PVE::Tools::split_list($data)) {
1555         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1556             $res->{$1} = 1;
1557         } else {
1558             die "invalid hotplug feature '$feature'\n";
1559         }
1560     }
1561     return $res;
1562 }
1563
1564 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1565 sub pve_verify_hotplug_features {
1566     my ($value, $noerr) = @_;
1567
1568     return $value if parse_hotplug_features($value);
1569
1570     return undef if $noerr;
1571
1572     die "unable to parse hotplug option\n";
1573 }
1574
1575 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1576 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1577 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1578 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1579 #        [,iothread=on][,serial=serial][,model=model]
1580
1581 sub parse_drive {
1582     my ($key, $data) = @_;
1583
1584     my ($interface, $index);
1585
1586     if ($key =~ m/^([^\d]+)(\d+)$/) {
1587         $interface = $1;
1588         $index = $2;
1589     } else {
1590         return undef;
1591     }
1592
1593     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1594                                      : $confdesc->{$key}->{format};
1595     if (!$desc) {
1596         warn "invalid drive key: $key\n";
1597         return undef;
1598     }
1599     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1600     return undef if !$res;
1601     $res->{interface} = $interface;
1602     $res->{index} = $index;
1603
1604     my $error = 0;
1605     foreach my $opt (qw(bps bps_rd bps_wr)) {
1606         if (my $bps = defined(delete $res->{$opt})) {
1607             if (defined($res->{"m$opt"})) {
1608                 warn "both $opt and m$opt specified\n";
1609                 ++$error;
1610                 next;
1611             }
1612             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1613         }
1614     }
1615
1616     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1617     for my $requirement (
1618         [mbps_max => 'mbps'],
1619         [mbps_rd_max => 'mbps_rd'],
1620         [mbps_wr_max => 'mbps_wr'],
1621         [miops_max => 'miops'],
1622         [miops_rd_max => 'miops_rd'],
1623         [miops_wr_max => 'miops_wr'],
1624         [bps_max_length => 'mbps_max'],
1625         [bps_rd_max_length => 'mbps_rd_max'],
1626         [bps_wr_max_length => 'mbps_wr_max'],
1627         [iops_max_length => 'iops_max'],
1628         [iops_rd_max_length => 'iops_rd_max'],
1629         [iops_wr_max_length => 'iops_wr_max']) {
1630         my ($option, $requires) = @$requirement;
1631         if ($res->{$option} && !$res->{$requires}) {
1632             warn "$option requires $requires\n";
1633             ++$error;
1634         }
1635     }
1636
1637     return undef if $error;
1638
1639     return undef if $res->{mbps_rd} && $res->{mbps};
1640     return undef if $res->{mbps_wr} && $res->{mbps};
1641     return undef if $res->{iops_rd} && $res->{iops};
1642     return undef if $res->{iops_wr} && $res->{iops};
1643
1644     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1645         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1646         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1647         return undef if $res->{interface} eq 'virtio';
1648     }
1649
1650     if (my $size = $res->{size}) {
1651         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1652     }
1653
1654     return $res;
1655 }
1656
1657 sub print_drive {
1658     my ($vmid, $drive) = @_;
1659     my $data = { %$drive };
1660     delete $data->{$_} for qw(index interface);
1661     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1662 }
1663
1664 sub scsi_inquiry {
1665     my($fh, $noerr) = @_;
1666
1667     my $SG_IO = 0x2285;
1668     my $SG_GET_VERSION_NUM = 0x2282;
1669
1670     my $versionbuf = "\x00" x 8;
1671     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1672     if (!$ret) {
1673         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1674         return undef;
1675     }
1676     my $version = unpack("I", $versionbuf);
1677     if ($version < 30000) {
1678         die "scsi generic interface too old\n"  if !$noerr;
1679         return undef;
1680     }
1681
1682     my $buf = "\x00" x 36;
1683     my $sensebuf = "\x00" x 8;
1684     my $cmd = pack("C x3 C x1", 0x12, 36);
1685
1686     # see /usr/include/scsi/sg.h
1687     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1688
1689     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1690                       length($sensebuf), 0, length($buf), $buf,
1691                       $cmd, $sensebuf, 6000);
1692
1693     $ret = ioctl($fh, $SG_IO, $packet);
1694     if (!$ret) {
1695         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1696         return undef;
1697     }
1698
1699     my @res = unpack($sg_io_hdr_t, $packet);
1700     if ($res[17] || $res[18]) {
1701         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1702         return undef;
1703     }
1704
1705     my $res = {};
1706     (my $byte0, my $byte1, $res->{vendor},
1707      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1708
1709     $res->{removable} = $byte1 & 128 ? 1 : 0;
1710     $res->{type} = $byte0 & 31;
1711
1712     return $res;
1713 }
1714
1715 sub path_is_scsi {
1716     my ($path) = @_;
1717
1718     my $fh = IO::File->new("+<$path") || return undef;
1719     my $res = scsi_inquiry($fh, 1);
1720     close($fh);
1721
1722     return $res;
1723 }
1724
1725 sub machine_type_is_q35 {
1726     my ($conf) = @_;
1727
1728     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1729 }
1730
1731 sub print_tabletdevice_full {
1732     my ($conf, $arch) = @_;
1733
1734     my $q35 = machine_type_is_q35($conf);
1735
1736     # we use uhci for old VMs because tablet driver was buggy in older qemu
1737     my $usbbus;
1738     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1739         $usbbus = 'ehci';
1740     } else {
1741         $usbbus = 'uhci';
1742     }
1743
1744     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1745 }
1746
1747 sub print_keyboarddevice_full {
1748     my ($conf, $arch, $machine) = @_;
1749
1750     return undef if $arch ne 'aarch64';
1751
1752     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1753 }
1754
1755 sub print_drivedevice_full {
1756     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1757
1758     my $device = '';
1759     my $maxdev = 0;
1760
1761     if ($drive->{interface} eq 'virtio') {
1762         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1763         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1764         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1765     } elsif ($drive->{interface} eq 'scsi') {
1766
1767         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1768         my $unit = $drive->{index} % $maxdev;
1769         my $devicetype = 'hd';
1770         my $path = '';
1771         if (drive_is_cdrom($drive)) {
1772             $devicetype = 'cd';
1773         } else {
1774             if ($drive->{file} =~ m|^/|) {
1775                 $path = $drive->{file};
1776                 if (my $info = path_is_scsi($path)) {
1777                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1778                         $devicetype = 'block';
1779                     } elsif ($info->{type} == 1) { # tape
1780                         $devicetype = 'generic';
1781                     }
1782                 }
1783             } else {
1784                  $path = PVE::Storage::path($storecfg, $drive->{file});
1785             }
1786
1787             if($path =~ m/^iscsi\:\/\//){
1788                 $devicetype = 'generic';
1789             }
1790         }
1791
1792         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1793             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1794         } else {
1795             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1796         }
1797
1798         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1799             $device .= ",rotation_rate=1";
1800         }
1801         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1802
1803     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1804         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1805         my $controller = int($drive->{index} / $maxdev);
1806         my $unit = $drive->{index} % $maxdev;
1807         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1808
1809         $device = "ide-$devicetype";
1810         if ($drive->{interface} eq 'ide') {
1811             $device .= ",bus=ide.$controller,unit=$unit";
1812         } else {
1813             $device .= ",bus=ahci$controller.$unit";
1814         }
1815         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1816
1817         if ($devicetype eq 'hd') {
1818             if (my $model = $drive->{model}) {
1819                 $model = URI::Escape::uri_unescape($model);
1820                 $device .= ",model=$model";
1821             }
1822             if ($drive->{ssd}) {
1823                 $device .= ",rotation_rate=1";
1824             }
1825         }
1826         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1827     } elsif ($drive->{interface} eq 'usb') {
1828         die "implement me";
1829         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1830     } else {
1831         die "unsupported interface type";
1832     }
1833
1834     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1835
1836     if (my $serial = $drive->{serial}) {
1837         $serial = URI::Escape::uri_unescape($serial);
1838         $device .= ",serial=$serial";
1839     }
1840
1841
1842     return $device;
1843 }
1844
1845 sub get_initiator_name {
1846     my $initiator;
1847
1848     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1849     while (defined(my $line = <$fh>)) {
1850         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1851         $initiator = $1;
1852         last;
1853     }
1854     $fh->close();
1855
1856     return $initiator;
1857 }
1858
1859 sub print_drive_full {
1860     my ($storecfg, $vmid, $drive) = @_;
1861
1862     my $path;
1863     my $volid = $drive->{file};
1864     my $format;
1865
1866     if (drive_is_cdrom($drive)) {
1867         $path = get_iso_path($storecfg, $vmid, $volid);
1868     } else {
1869         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1870         if ($storeid) {
1871             $path = PVE::Storage::path($storecfg, $volid);
1872             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1873             $format = qemu_img_format($scfg, $volname);
1874         } else {
1875             $path = $volid;
1876             $format = "raw";
1877         }
1878    }
1879
1880     my $opts = '';
1881     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1882     foreach my $o (@qemu_drive_options) {
1883         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1884     }
1885
1886     # snapshot only accepts on|off
1887     if (defined($drive->{snapshot})) {
1888         my $v = $drive->{snapshot} ? 'on' : 'off';
1889         $opts .= ",snapshot=$v";
1890     }
1891
1892     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1893         my ($dir, $qmpname) = @$type;
1894         if (my $v = $drive->{"mbps$dir"}) {
1895             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1896         }
1897         if (my $v = $drive->{"mbps${dir}_max"}) {
1898             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1899         }
1900         if (my $v = $drive->{"bps${dir}_max_length"}) {
1901             $opts .= ",throttling.bps$qmpname-max-length=$v";
1902         }
1903         if (my $v = $drive->{"iops${dir}"}) {
1904             $opts .= ",throttling.iops$qmpname=$v";
1905         }
1906         if (my $v = $drive->{"iops${dir}_max"}) {
1907             $opts .= ",throttling.iops$qmpname-max=$v";
1908         }
1909         if (my $v = $drive->{"iops${dir}_max_length"}) {
1910             $opts .= ",throttling.iops$qmpname-max-length=$v";
1911         }
1912     }
1913
1914     $opts .= ",format=$format" if $format && !$drive->{format};
1915
1916     my $cache_direct = 0;
1917
1918     if (my $cache = $drive->{cache}) {
1919         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1920     } elsif (!drive_is_cdrom($drive)) {
1921         $opts .= ",cache=none";
1922         $cache_direct = 1;
1923     }
1924
1925     # aio native works only with O_DIRECT
1926     if (!$drive->{aio}) {
1927         if($cache_direct) {
1928             $opts .= ",aio=native";
1929         } else {
1930             $opts .= ",aio=threads";
1931         }
1932     }
1933
1934     if (!drive_is_cdrom($drive)) {
1935         my $detectzeroes;
1936         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1937             $detectzeroes = 'off';
1938         } elsif ($drive->{discard}) {
1939             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1940         } else {
1941             # This used to be our default with discard not being specified:
1942             $detectzeroes = 'on';
1943         }
1944         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1945     }
1946
1947     my $pathinfo = $path ? "file=$path," : '';
1948
1949     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1950 }
1951
1952 sub print_netdevice_full {
1953     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1954
1955     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1956
1957     my $device = $net->{model};
1958     if ($net->{model} eq 'virtio') {
1959          $device = 'virtio-net-pci';
1960      };
1961
1962     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1963     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1964     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1965         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1966         my $vectors = $net->{queues} * 2 + 2;
1967         $tmpstr .= ",vectors=$vectors,mq=on";
1968     }
1969     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1970
1971     if ($use_old_bios_files) {
1972         my $romfile;
1973         if ($device eq 'virtio-net-pci') {
1974             $romfile = 'pxe-virtio.rom';
1975         } elsif ($device eq 'e1000') {
1976             $romfile = 'pxe-e1000.rom';
1977         } elsif ($device eq 'ne2k') {
1978             $romfile = 'pxe-ne2k_pci.rom';
1979         } elsif ($device eq 'pcnet') {
1980             $romfile = 'pxe-pcnet.rom';
1981         } elsif ($device eq 'rtl8139') {
1982             $romfile = 'pxe-rtl8139.rom';
1983         }
1984         $tmpstr .= ",romfile=$romfile" if $romfile;
1985     }
1986
1987     return $tmpstr;
1988 }
1989
1990 sub print_netdev_full {
1991     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1992
1993     my $i = '';
1994     if ($netid =~ m/^net(\d+)$/) {
1995         $i = int($1);
1996     }
1997
1998     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1999
2000     my $ifname = "tap${vmid}i$i";
2001
2002     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2003     die "interface name '$ifname' is too long (max 15 character)\n"
2004         if length($ifname) >= 16;
2005
2006     my $vhostparam = '';
2007     if (is_native($arch)) {
2008         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2009     }
2010
2011     my $vmname = $conf->{name} || "vm$vmid";
2012
2013     my $netdev = "";
2014     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2015
2016     if ($net->{bridge}) {
2017         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2018     } else {
2019         $netdev = "type=user,id=$netid,hostname=$vmname";
2020     }
2021
2022     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2023
2024     return $netdev;
2025 }
2026
2027
2028 sub print_cpu_device {
2029     my ($conf, $id) = @_;
2030
2031     my $kvm = $conf->{kvm} // 1;
2032     my $cpu = $kvm ? "kvm64" : "qemu64";
2033     if (my $cputype = $conf->{cpu}) {
2034         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2035             or die "Cannot parse cpu description: $cputype\n";
2036         $cpu = $cpuconf->{cputype};
2037     }
2038
2039     my $cores = $conf->{cores} || 1;
2040
2041     my $current_core = ($id - 1) % $cores;
2042     my $current_socket = int(($id - 1 - $current_core)/$cores);
2043
2044     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2045 }
2046
2047 my $vga_map = {
2048     'cirrus' => 'cirrus-vga',
2049     'std' => 'VGA',
2050     'vmware' => 'vmware-svga',
2051     'virtio' => 'virtio-vga',
2052 };
2053
2054 sub print_vga_device {
2055     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2056
2057     my $type = $vga_map->{$vga->{type}};
2058     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2059         $type = 'virtio-gpu';
2060     }
2061     my $vgamem_mb = $vga->{memory};
2062     if ($qxlnum) {
2063         $type = $id ? 'qxl' : 'qxl-vga';
2064     }
2065     die "no devicetype for $vga->{type}\n" if !$type;
2066
2067     my $memory = "";
2068     if ($vgamem_mb) {
2069         if ($vga->{type} eq 'virtio') {
2070             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2071             $memory = ",max_hostmem=$bytes";
2072         } elsif ($qxlnum) {
2073             # from https://www.spice-space.org/multiple-monitors.html
2074             $memory = ",vgamem_mb=$vga->{memory}";
2075             my $ram = $vgamem_mb * 4;
2076             my $vram = $vgamem_mb * 2;
2077             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2078         } else {
2079             $memory = ",vgamem_mb=$vga->{memory}";
2080         }
2081     } elsif ($qxlnum && $id) {
2082         $memory = ",ram_size=67108864,vram_size=33554432";
2083     }
2084
2085     my $q35 = machine_type_is_q35($conf);
2086     my $vgaid = "vga" . ($id // '');
2087     my $pciaddr;
2088
2089     if ($q35 && $vgaid eq 'vga') {
2090         # the first display uses pcie.0 bus on q35 machines
2091         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2092     } else {
2093         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2094     }
2095
2096     return "$type,id=${vgaid}${memory}${pciaddr}";
2097 }
2098
2099 sub drive_is_cloudinit {
2100     my ($drive) = @_;
2101     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2102 }
2103
2104 sub drive_is_cdrom {
2105     my ($drive, $exclude_cloudinit) = @_;
2106
2107     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2108
2109     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2110
2111 }
2112
2113 sub parse_number_sets {
2114     my ($set) = @_;
2115     my $res = [];
2116     foreach my $part (split(/;/, $set)) {
2117         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2118             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2119             push @$res, [ $1, $2 ];
2120         } else {
2121             die "invalid range: $part\n";
2122         }
2123     }
2124     return $res;
2125 }
2126
2127 sub parse_numa {
2128     my ($data) = @_;
2129
2130     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2131     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2132     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2133     return $res;
2134 }
2135
2136 sub parse_hostpci {
2137     my ($value) = @_;
2138
2139     return undef if !$value;
2140
2141     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2142
2143     my @idlist = split(/;/, $res->{host});
2144     delete $res->{host};
2145     foreach my $id (@idlist) {
2146         if ($id =~ m/\./) { # full id 00:00.1
2147             push @{$res->{pciid}}, {
2148                 id => $id,
2149             };
2150         } else { # partial id 00:00
2151             $res->{pciid} = PVE::SysFSTools::lspci($id);
2152         }
2153     }
2154     return $res;
2155 }
2156
2157 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2158 sub parse_net {
2159     my ($data) = @_;
2160
2161     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2162     if ($@) {
2163         warn $@;
2164         return undef;
2165     }
2166     if (!defined($res->{macaddr})) {
2167         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2168         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2169     }
2170     return $res;
2171 }
2172
2173 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2174 sub parse_ipconfig {
2175     my ($data) = @_;
2176
2177     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2178     if ($@) {
2179         warn $@;
2180         return undef;
2181     }
2182
2183     if ($res->{gw} && !$res->{ip}) {
2184         warn 'gateway specified without specifying an IP address';
2185         return undef;
2186     }
2187     if ($res->{gw6} && !$res->{ip6}) {
2188         warn 'IPv6 gateway specified without specifying an IPv6 address';
2189         return undef;
2190     }
2191     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2192         warn 'gateway specified together with DHCP';
2193         return undef;
2194     }
2195     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2196         # gw6 + auto/dhcp
2197         warn "IPv6 gateway specified together with $res->{ip6} address";
2198         return undef;
2199     }
2200
2201     if (!$res->{ip} && !$res->{ip6}) {
2202         return { ip => 'dhcp', ip6 => 'dhcp' };
2203     }
2204
2205     return $res;
2206 }
2207
2208 sub print_net {
2209     my $net = shift;
2210
2211     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2212 }
2213
2214 sub add_random_macs {
2215     my ($settings) = @_;
2216
2217     foreach my $opt (keys %$settings) {
2218         next if $opt !~ m/^net(\d+)$/;
2219         my $net = parse_net($settings->{$opt});
2220         next if !$net;
2221         $settings->{$opt} = print_net($net);
2222     }
2223 }
2224
2225 sub vm_is_volid_owner {
2226     my ($storecfg, $vmid, $volid) = @_;
2227
2228     if ($volid !~  m|^/|) {
2229         my ($path, $owner);
2230         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2231         if ($owner && ($owner == $vmid)) {
2232             return 1;
2233         }
2234     }
2235
2236     return undef;
2237 }
2238
2239 sub split_flagged_list {
2240     my $text = shift || '';
2241     $text =~ s/[,;]/ /g;
2242     $text =~ s/^\s+//;
2243     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2244 }
2245
2246 sub join_flagged_list {
2247     my ($how, $lst) = @_;
2248     join $how, map { $lst->{$_} . $_ } keys %$lst;
2249 }
2250
2251 sub vmconfig_delete_pending_option {
2252     my ($conf, $key, $force) = @_;
2253
2254     delete $conf->{pending}->{$key};
2255     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2256     $pending_delete_hash->{$key} = $force ? '!' : '';
2257     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2258 }
2259
2260 sub vmconfig_undelete_pending_option {
2261     my ($conf, $key) = @_;
2262
2263     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2264     delete $pending_delete_hash->{$key};
2265
2266     if (%$pending_delete_hash) {
2267         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2268     } else {
2269         delete $conf->{pending}->{delete};
2270     }
2271 }
2272
2273 sub vmconfig_register_unused_drive {
2274     my ($storecfg, $vmid, $conf, $drive) = @_;
2275
2276     if (drive_is_cloudinit($drive)) {
2277         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2278         warn $@ if $@;
2279     } elsif (!drive_is_cdrom($drive)) {
2280         my $volid = $drive->{file};
2281         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2282             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2283         }
2284     }
2285 }
2286
2287 sub vmconfig_cleanup_pending {
2288     my ($conf) = @_;
2289
2290     # remove pending changes when nothing changed
2291     my $changes;
2292     foreach my $opt (keys %{$conf->{pending}}) {
2293         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2294             $changes = 1;
2295             delete $conf->{pending}->{$opt};
2296         }
2297     }
2298
2299     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2300     my $pending_delete_hash = {};
2301     while (my ($opt, $force) = each %$current_delete_hash) {
2302         if (defined($conf->{$opt})) {
2303             $pending_delete_hash->{$opt} = $force;
2304         } else {
2305             $changes = 1;
2306         }
2307     }
2308
2309     if (%$pending_delete_hash) {
2310         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2311     } else {
2312         delete $conf->{pending}->{delete};
2313     }
2314
2315     return $changes;
2316 }
2317
2318 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2319 my $smbios1_fmt = {
2320     uuid => {
2321         type => 'string',
2322         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2323         format_description => 'UUID',
2324         description => "Set SMBIOS1 UUID.",
2325         optional => 1,
2326     },
2327     version => {
2328         type => 'string',
2329         pattern => '\S+',
2330         format_description => 'string',
2331         description => "Set SMBIOS1 version.",
2332         optional => 1,
2333     },
2334     serial => {
2335         type => 'string',
2336         pattern => '\S+',
2337         format_description => 'string',
2338         description => "Set SMBIOS1 serial number.",
2339         optional => 1,
2340     },
2341     manufacturer => {
2342         type => 'string',
2343         pattern => '\S+',
2344         format_description => 'string',
2345         description => "Set SMBIOS1 manufacturer.",
2346         optional => 1,
2347     },
2348     product => {
2349         type => 'string',
2350         pattern => '\S+',
2351         format_description => 'string',
2352         description => "Set SMBIOS1 product ID.",
2353         optional => 1,
2354     },
2355     sku => {
2356         type => 'string',
2357         pattern => '\S+',
2358         format_description => 'string',
2359         description => "Set SMBIOS1 SKU string.",
2360         optional => 1,
2361     },
2362     family => {
2363         type => 'string',
2364         pattern => '\S+',
2365         format_description => 'string',
2366         description => "Set SMBIOS1 family string.",
2367         optional => 1,
2368     },
2369 };
2370
2371 sub parse_smbios1 {
2372     my ($data) = @_;
2373
2374     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2375     warn $@ if $@;
2376     return $res;
2377 }
2378
2379 sub print_smbios1 {
2380     my ($smbios1) = @_;
2381     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2382 }
2383
2384 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2385
2386 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2387 sub verify_bootdisk {
2388     my ($value, $noerr) = @_;
2389
2390     return $value if is_valid_drivename($value);
2391
2392     return undef if $noerr;
2393
2394     die "invalid boot disk '$value'\n";
2395 }
2396
2397 sub parse_watchdog {
2398     my ($value) = @_;
2399
2400     return undef if !$value;
2401
2402     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2403     warn $@ if $@;
2404     return $res;
2405 }
2406
2407 sub parse_guest_agent {
2408     my ($value) = @_;
2409
2410     return {} if !defined($value->{agent});
2411
2412     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2413     warn $@ if $@;
2414
2415     # if the agent is disabled ignore the other potentially set properties
2416     return {} if !$res->{enabled};
2417     return $res;
2418 }
2419
2420 sub parse_vga {
2421     my ($value) = @_;
2422
2423     return {} if !$value;
2424     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2425     warn $@ if $@;
2426     return $res;
2427 }
2428
2429 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2430 sub verify_usb_device {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if parse_usb_device($value);
2434
2435     return undef if $noerr;
2436
2437     die "unable to parse usb device\n";
2438 }
2439
2440 # add JSON properties for create and set function
2441 sub json_config_properties {
2442     my $prop = shift;
2443
2444     foreach my $opt (keys %$confdesc) {
2445         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2446         $prop->{$opt} = $confdesc->{$opt};
2447     }
2448
2449     return $prop;
2450 }
2451
2452 # return copy of $confdesc_cloudinit to generate documentation
2453 sub cloudinit_config_properties {
2454
2455     return dclone($confdesc_cloudinit);
2456 }
2457
2458 sub check_type {
2459     my ($key, $value) = @_;
2460
2461     die "unknown setting '$key'\n" if !$confdesc->{$key};
2462
2463     my $type = $confdesc->{$key}->{type};
2464
2465     if (!defined($value)) {
2466         die "got undefined value\n";
2467     }
2468
2469     if ($value =~ m/[\n\r]/) {
2470         die "property contains a line feed\n";
2471     }
2472
2473     if ($type eq 'boolean') {
2474         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2475         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2476         die "type check ('boolean') failed - got '$value'\n";
2477     } elsif ($type eq 'integer') {
2478         return int($1) if $value =~ m/^(\d+)$/;
2479         die "type check ('integer') failed - got '$value'\n";
2480     } elsif ($type eq 'number') {
2481         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2482         die "type check ('number') failed - got '$value'\n";
2483     } elsif ($type eq 'string') {
2484         if (my $fmt = $confdesc->{$key}->{format}) {
2485             PVE::JSONSchema::check_format($fmt, $value);
2486             return $value;
2487         }
2488         $value =~ s/^\"(.*)\"$/$1/;
2489         return $value;
2490     } else {
2491         die "internal error"
2492     }
2493 }
2494
2495 sub touch_config {
2496     my ($vmid) = @_;
2497
2498     my $conf = PVE::QemuConfig->config_file($vmid);
2499     utime undef, undef, $conf;
2500 }
2501
2502 sub destroy_vm {
2503     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2504
2505     my $conffile = PVE::QemuConfig->config_file($vmid);
2506
2507     my $conf = PVE::QemuConfig->load_config($vmid);
2508
2509     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2510
2511     if ($conf->{template}) {
2512         # check if any base image is still used by a linked clone
2513         foreach_drive($conf, sub {
2514                 my ($ds, $drive) = @_;
2515
2516                 return if drive_is_cdrom($drive);
2517
2518                 my $volid = $drive->{file};
2519
2520                 return if !$volid || $volid =~ m|^/|;
2521
2522                 die "base volume '$volid' is still in use by linked cloned\n"
2523                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2524
2525         });
2526     }
2527
2528     # only remove disks owned by this VM
2529     foreach_drive($conf, sub {
2530         my ($ds, $drive) = @_;
2531
2532         return if drive_is_cdrom($drive, 1);
2533
2534         my $volid = $drive->{file};
2535
2536         return if !$volid || $volid =~ m|^/|;
2537
2538         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2539         return if !$path || !$owner || ($owner != $vmid);
2540
2541         eval {
2542             PVE::Storage::vdisk_free($storecfg, $volid);
2543         };
2544         warn "Could not remove disk '$volid', check manually: $@" if $@;
2545
2546     });
2547
2548     if ($keep_empty_config) {
2549         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2550     } else {
2551         unlink $conffile;
2552     }
2553
2554     # also remove unused disk
2555     eval {
2556         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2557
2558         eval {
2559             PVE::Storage::foreach_volid($dl, sub {
2560                 my ($volid, $sid, $volname, $d) = @_;
2561                 PVE::Storage::vdisk_free($storecfg, $volid);
2562             });
2563         };
2564         warn $@ if $@;
2565
2566     };
2567     warn $@ if $@;
2568 }
2569
2570 sub parse_vm_config {
2571     my ($filename, $raw) = @_;
2572
2573     return undef if !defined($raw);
2574
2575     my $res = {
2576         digest => Digest::SHA::sha1_hex($raw),
2577         snapshots => {},
2578         pending => {},
2579     };
2580
2581     $filename =~ m|/qemu-server/(\d+)\.conf$|
2582         || die "got strange filename '$filename'";
2583
2584     my $vmid = $1;
2585
2586     my $conf = $res;
2587     my $descr;
2588     my $section = '';
2589
2590     my @lines = split(/\n/, $raw);
2591     foreach my $line (@lines) {
2592         next if $line =~ m/^\s*$/;
2593
2594         if ($line =~ m/^\[PENDING\]\s*$/i) {
2595             $section = 'pending';
2596             if (defined($descr)) {
2597                 $descr =~ s/\s+$//;
2598                 $conf->{description} = $descr;
2599             }
2600             $descr = undef;
2601             $conf = $res->{$section} = {};
2602             next;
2603
2604         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2605             $section = $1;
2606             if (defined($descr)) {
2607                 $descr =~ s/\s+$//;
2608                 $conf->{description} = $descr;
2609             }
2610             $descr = undef;
2611             $conf = $res->{snapshots}->{$section} = {};
2612             next;
2613         }
2614
2615         if ($line =~ m/^\#(.*)\s*$/) {
2616             $descr = '' if !defined($descr);
2617             $descr .= PVE::Tools::decode_text($1) . "\n";
2618             next;
2619         }
2620
2621         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2622             $descr = '' if !defined($descr);
2623             $descr .= PVE::Tools::decode_text($2);
2624         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2625             $conf->{snapstate} = $1;
2626         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2627             my $key = $1;
2628             my $value = $2;
2629             $conf->{$key} = $value;
2630         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2631             my $value = $1;
2632             if ($section eq 'pending') {
2633                 $conf->{delete} = $value; # we parse this later
2634             } else {
2635                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2636             }
2637         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2638             my $key = $1;
2639             my $value = $2;
2640             eval { $value = check_type($key, $value); };
2641             if ($@) {
2642                 warn "vm $vmid - unable to parse value of '$key' - $@";
2643             } else {
2644                 $key = 'ide2' if $key eq 'cdrom';
2645                 my $fmt = $confdesc->{$key}->{format};
2646                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2647                     my $v = parse_drive($key, $value);
2648                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2649                         $v->{file} = $volid;
2650                         $value = print_drive($vmid, $v);
2651                     } else {
2652                         warn "vm $vmid - unable to parse value of '$key'\n";
2653                         next;
2654                     }
2655                 }
2656
2657                 $conf->{$key} = $value;
2658             }
2659         }
2660     }
2661
2662     if (defined($descr)) {
2663         $descr =~ s/\s+$//;
2664         $conf->{description} = $descr;
2665     }
2666     delete $res->{snapstate}; # just to be sure
2667
2668     return $res;
2669 }
2670
2671 sub write_vm_config {
2672     my ($filename, $conf) = @_;
2673
2674     delete $conf->{snapstate}; # just to be sure
2675
2676     if ($conf->{cdrom}) {
2677         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2678         $conf->{ide2} = $conf->{cdrom};
2679         delete $conf->{cdrom};
2680     }
2681
2682     # we do not use 'smp' any longer
2683     if ($conf->{sockets}) {
2684         delete $conf->{smp};
2685     } elsif ($conf->{smp}) {
2686         $conf->{sockets} = $conf->{smp};
2687         delete $conf->{cores};
2688         delete $conf->{smp};
2689     }
2690
2691     my $used_volids = {};
2692
2693     my $cleanup_config = sub {
2694         my ($cref, $pending, $snapname) = @_;
2695
2696         foreach my $key (keys %$cref) {
2697             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2698                 $key eq 'snapstate' || $key eq 'pending';
2699             my $value = $cref->{$key};
2700             if ($key eq 'delete') {
2701                 die "propertry 'delete' is only allowed in [PENDING]\n"
2702                     if !$pending;
2703                 # fixme: check syntax?
2704                 next;
2705             }
2706             eval { $value = check_type($key, $value); };
2707             die "unable to parse value of '$key' - $@" if $@;
2708
2709             $cref->{$key} = $value;
2710
2711             if (!$snapname && is_valid_drivename($key)) {
2712                 my $drive = parse_drive($key, $value);
2713                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2714             }
2715         }
2716     };
2717
2718     &$cleanup_config($conf);
2719
2720     &$cleanup_config($conf->{pending}, 1);
2721
2722     foreach my $snapname (keys %{$conf->{snapshots}}) {
2723         die "internal error" if $snapname eq 'pending';
2724         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2725     }
2726
2727     # remove 'unusedX' settings if we re-add a volume
2728     foreach my $key (keys %$conf) {
2729         my $value = $conf->{$key};
2730         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2731             delete $conf->{$key};
2732         }
2733     }
2734
2735     my $generate_raw_config = sub {
2736         my ($conf, $pending) = @_;
2737
2738         my $raw = '';
2739
2740         # add description as comment to top of file
2741         if (defined(my $descr = $conf->{description})) {
2742             if ($descr) {
2743                 foreach my $cl (split(/\n/, $descr)) {
2744                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2745                 }
2746             } else {
2747                 $raw .= "#\n" if $pending;
2748             }
2749         }
2750
2751         foreach my $key (sort keys %$conf) {
2752             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2753             $raw .= "$key: $conf->{$key}\n";
2754         }
2755         return $raw;
2756     };
2757
2758     my $raw = &$generate_raw_config($conf);
2759
2760     if (scalar(keys %{$conf->{pending}})){
2761         $raw .= "\n[PENDING]\n";
2762         $raw .= &$generate_raw_config($conf->{pending}, 1);
2763     }
2764
2765     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2766         $raw .= "\n[$snapname]\n";
2767         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2768     }
2769
2770     return $raw;
2771 }
2772
2773 sub load_defaults {
2774
2775     my $res = {};
2776
2777     # we use static defaults from our JSON schema configuration
2778     foreach my $key (keys %$confdesc) {
2779         if (defined(my $default = $confdesc->{$key}->{default})) {
2780             $res->{$key} = $default;
2781         }
2782     }
2783
2784     return $res;
2785 }
2786
2787 sub config_list {
2788     my $vmlist = PVE::Cluster::get_vmlist();
2789     my $res = {};
2790     return $res if !$vmlist || !$vmlist->{ids};
2791     my $ids = $vmlist->{ids};
2792
2793     foreach my $vmid (keys %$ids) {
2794         my $d = $ids->{$vmid};
2795         next if !$d->{node} || $d->{node} ne $nodename;
2796         next if !$d->{type} || $d->{type} ne 'qemu';
2797         $res->{$vmid}->{exists} = 1;
2798     }
2799     return $res;
2800 }
2801
2802 # test if VM uses local resources (to prevent migration)
2803 sub check_local_resources {
2804     my ($conf, $noerr) = @_;
2805
2806     my $loc_res = 0;
2807
2808     $loc_res = 1 if $conf->{hostusb}; # old syntax
2809     $loc_res = 1 if $conf->{hostpci}; # old syntax
2810
2811     foreach my $k (keys %$conf) {
2812         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2813         # sockets are safe: they will recreated be on the target side post-migrate
2814         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2815         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2816     }
2817
2818     die "VM uses local resources\n" if $loc_res && !$noerr;
2819
2820     return $loc_res;
2821 }
2822
2823 # check if used storages are available on all nodes (use by migrate)
2824 sub check_storage_availability {
2825     my ($storecfg, $conf, $node) = @_;
2826
2827     foreach_drive($conf, sub {
2828         my ($ds, $drive) = @_;
2829
2830         my $volid = $drive->{file};
2831         return if !$volid;
2832
2833         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2834         return if !$sid;
2835
2836         # check if storage is available on both nodes
2837         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2838         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2839    });
2840 }
2841
2842 # list nodes where all VM images are available (used by has_feature API)
2843 sub shared_nodes {
2844     my ($conf, $storecfg) = @_;
2845
2846     my $nodelist = PVE::Cluster::get_nodelist();
2847     my $nodehash = { map { $_ => 1 } @$nodelist };
2848     my $nodename = PVE::INotify::nodename();
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         if ($storeid) {
2858             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2859             if ($scfg->{disable}) {
2860                 $nodehash = {};
2861             } elsif (my $avail = $scfg->{nodes}) {
2862                 foreach my $node (keys %$nodehash) {
2863                     delete $nodehash->{$node} if !$avail->{$node};
2864                 }
2865             } elsif (!$scfg->{shared}) {
2866                 foreach my $node (keys %$nodehash) {
2867                     delete $nodehash->{$node} if $node ne $nodename
2868                 }
2869             }
2870         }
2871     });
2872
2873     return $nodehash
2874 }
2875
2876 sub check_cmdline {
2877     my ($pidfile, $pid) = @_;
2878
2879     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2880     if (defined($fh)) {
2881         my $line = <$fh>;
2882         $fh->close;
2883         return undef if !$line;
2884         my @param = split(/\0/, $line);
2885
2886         my $cmd = $param[0];
2887         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2888
2889         for (my $i = 0; $i < scalar (@param); $i++) {
2890             my $p = $param[$i];
2891             next if !$p;
2892             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2893                 my $p = $param[$i+1];
2894                 return 1 if $p && ($p eq $pidfile);
2895                 return undef;
2896             }
2897         }
2898     }
2899     return undef;
2900 }
2901
2902 sub check_running {
2903     my ($vmid, $nocheck, $node) = @_;
2904
2905     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2906
2907     die "unable to find configuration file for VM $vmid - no such machine\n"
2908         if !$nocheck && ! -f $filename;
2909
2910     my $pidfile = pidfile_name($vmid);
2911
2912     if (my $fd = IO::File->new("<$pidfile")) {
2913         my $st = stat($fd);
2914         my $line = <$fd>;
2915         close($fd);
2916
2917         my $mtime = $st->mtime;
2918         if ($mtime > time()) {
2919             warn "file '$filename' modified in future\n";
2920         }
2921
2922         if ($line =~ m/^(\d+)$/) {
2923             my $pid = $1;
2924             if (check_cmdline($pidfile, $pid)) {
2925                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2926                     return $pid;
2927                 }
2928             }
2929         }
2930     }
2931
2932     return undef;
2933 }
2934
2935 sub vzlist {
2936
2937     my $vzlist = config_list();
2938
2939     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2940
2941     while (defined(my $de = $fd->read)) {
2942         next if $de !~ m/^(\d+)\.pid$/;
2943         my $vmid = $1;
2944         next if !defined($vzlist->{$vmid});
2945         if (my $pid = check_running($vmid)) {
2946             $vzlist->{$vmid}->{pid} = $pid;
2947         }
2948     }
2949
2950     return $vzlist;
2951 }
2952
2953 sub disksize {
2954     my ($storecfg, $conf) = @_;
2955
2956     my $bootdisk = $conf->{bootdisk};
2957     return undef if !$bootdisk;
2958     return undef if !is_valid_drivename($bootdisk);
2959
2960     return undef if !$conf->{$bootdisk};
2961
2962     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2963     return undef if !defined($drive);
2964
2965     return undef if drive_is_cdrom($drive);
2966
2967     my $volid = $drive->{file};
2968     return undef if !$volid;
2969
2970     return $drive->{size};
2971 }
2972
2973 our $vmstatus_return_properties = {
2974     vmid => get_standard_option('pve-vmid'),
2975     status => {
2976         description => "Qemu process status.",
2977         type => 'string',
2978         enum => ['stopped', 'running'],
2979     },
2980     maxmem => {
2981         description => "Maximum memory in bytes.",
2982         type => 'integer',
2983         optional => 1,
2984         renderer => 'bytes',
2985     },
2986     maxdisk => {
2987         description => "Root disk size in bytes.",
2988         type => 'integer',
2989         optional => 1,
2990         renderer => 'bytes',
2991     },
2992     name => {
2993         description => "VM name.",
2994         type => 'string',
2995         optional => 1,
2996     },
2997     qmpstatus => {
2998         description => "Qemu QMP agent status.",
2999         type => 'string',
3000         optional => 1,
3001     },
3002     pid => {
3003         description => "PID of running qemu process.",
3004         type => 'integer',
3005         optional => 1,
3006     },
3007     uptime => {
3008         description => "Uptime.",
3009         type => 'integer',
3010         optional => 1,
3011         renderer => 'duration',
3012     },
3013     cpus => {
3014         description => "Maximum usable CPUs.",
3015         type => 'number',
3016         optional => 1,
3017     },
3018 };
3019
3020 my $last_proc_pid_stat;
3021
3022 # get VM status information
3023 # This must be fast and should not block ($full == false)
3024 # We only query KVM using QMP if $full == true (this can be slow)
3025 sub vmstatus {
3026     my ($opt_vmid, $full) = @_;
3027
3028     my $res = {};
3029
3030     my $storecfg = PVE::Storage::config();
3031
3032     my $list = vzlist();
3033     my $defaults = load_defaults();
3034
3035     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3036
3037     my $cpucount = $cpuinfo->{cpus} || 1;
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041
3042         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3043         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3044
3045         my $d = { vmid => $vmid };
3046         $d->{pid} = $list->{$vmid}->{pid};
3047
3048         # fixme: better status?
3049         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3050
3051         my $size = disksize($storecfg, $conf);
3052         if (defined($size)) {
3053             $d->{disk} = 0; # no info available
3054             $d->{maxdisk} = $size;
3055         } else {
3056             $d->{disk} = 0;
3057             $d->{maxdisk} = 0;
3058         }
3059
3060         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3061             * ($conf->{cores} || $defaults->{cores});
3062         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3063         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3064
3065         $d->{name} = $conf->{name} || "VM $vmid";
3066         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3067             : $defaults->{memory}*(1024*1024);
3068
3069         if ($conf->{balloon}) {
3070             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3071             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3072                 : $defaults->{shares};
3073         }
3074
3075         $d->{uptime} = 0;
3076         $d->{cpu} = 0;
3077         $d->{mem} = 0;
3078
3079         $d->{netout} = 0;
3080         $d->{netin} = 0;
3081
3082         $d->{diskread} = 0;
3083         $d->{diskwrite} = 0;
3084
3085         $d->{template} = PVE::QemuConfig->is_template($conf);
3086
3087         $d->{serial} = 1 if conf_has_serial($conf);
3088
3089         $res->{$vmid} = $d;
3090     }
3091
3092     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3093     foreach my $dev (keys %$netdev) {
3094         next if $dev !~ m/^tap([1-9]\d*)i/;
3095         my $vmid = $1;
3096         my $d = $res->{$vmid};
3097         next if !$d;
3098
3099         $d->{netout} += $netdev->{$dev}->{receive};
3100         $d->{netin} += $netdev->{$dev}->{transmit};
3101
3102         if ($full) {
3103             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3104             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3105         }
3106
3107     }
3108
3109     my $ctime = gettimeofday;
3110
3111     foreach my $vmid (keys %$list) {
3112
3113         my $d = $res->{$vmid};
3114         my $pid = $d->{pid};
3115         next if !$pid;
3116
3117         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3118         next if !$pstat; # not running
3119
3120         my $used = $pstat->{utime} + $pstat->{stime};
3121
3122         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3123
3124         if ($pstat->{vsize}) {
3125             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3126         }
3127
3128         my $old = $last_proc_pid_stat->{$pid};
3129         if (!$old) {
3130             $last_proc_pid_stat->{$pid} = {
3131                 time => $ctime,
3132                 used => $used,
3133                 cpu => 0,
3134             };
3135             next;
3136         }
3137
3138         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3139
3140         if ($dtime > 1000) {
3141             my $dutime = $used -  $old->{used};
3142
3143             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3144             $last_proc_pid_stat->{$pid} = {
3145                 time => $ctime,
3146                 used => $used,
3147                 cpu => $d->{cpu},
3148             };
3149         } else {
3150             $d->{cpu} = $old->{cpu};
3151         }
3152     }
3153
3154     return $res if !$full;
3155
3156     my $qmpclient = PVE::QMPClient->new();
3157
3158     my $ballooncb = sub {
3159         my ($vmid, $resp) = @_;
3160
3161         my $info = $resp->{'return'};
3162         return if !$info->{max_mem};
3163
3164         my $d = $res->{$vmid};
3165
3166         # use memory assigned to VM
3167         $d->{maxmem} = $info->{max_mem};
3168         $d->{balloon} = $info->{actual};
3169
3170         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3171             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3172             $d->{freemem} = $info->{free_mem};
3173         }
3174
3175         $d->{ballooninfo} = $info;
3176     };
3177
3178     my $blockstatscb = sub {
3179         my ($vmid, $resp) = @_;
3180         my $data = $resp->{'return'} || [];
3181         my $totalrdbytes = 0;
3182         my $totalwrbytes = 0;
3183
3184         for my $blockstat (@$data) {
3185             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3186             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3187
3188             $blockstat->{device} =~ s/drive-//;
3189             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3190         }
3191         $res->{$vmid}->{diskread} = $totalrdbytes;
3192         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3193     };
3194
3195     my $statuscb = sub {
3196         my ($vmid, $resp) = @_;
3197
3198         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3199         # this fails if ballon driver is not loaded, so this must be
3200         # the last commnand (following command are aborted if this fails).
3201         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3202
3203         my $status = 'unknown';
3204         if (!defined($status = $resp->{'return'}->{status})) {
3205             warn "unable to get VM status\n";
3206             return;
3207         }
3208
3209         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3210     };
3211
3212     foreach my $vmid (keys %$list) {
3213         next if $opt_vmid && ($vmid ne $opt_vmid);
3214         next if !$res->{$vmid}->{pid}; # not running
3215         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3216     }
3217
3218     $qmpclient->queue_execute(undef, 2);
3219
3220     foreach my $vmid (keys %$list) {
3221         next if $opt_vmid && ($vmid ne $opt_vmid);
3222         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3223     }
3224
3225     return $res;
3226 }
3227
3228 sub foreach_drive {
3229     my ($conf, $func, @param) = @_;
3230
3231     foreach my $ds (valid_drive_names()) {
3232         next if !defined($conf->{$ds});
3233
3234         my $drive = parse_drive($ds, $conf->{$ds});
3235         next if !$drive;
3236
3237         &$func($ds, $drive, @param);
3238     }
3239 }
3240
3241 sub foreach_volid {
3242     my ($conf, $func, @param) = @_;
3243
3244     my $volhash = {};
3245
3246     my $test_volid = sub {
3247         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3248
3249         return if !$volid;
3250
3251         $volhash->{$volid}->{cdrom} //= 1;
3252         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3253
3254         $volhash->{$volid}->{replicate} //= 0;
3255         $volhash->{$volid}->{replicate} = 1 if $replicate;
3256
3257         $volhash->{$volid}->{shared} //= 0;
3258         $volhash->{$volid}->{shared} = 1 if $shared;
3259
3260         $volhash->{$volid}->{referenced_in_config} //= 0;
3261         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3262
3263         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3264             if defined($snapname);
3265     };
3266
3267     foreach_drive($conf, sub {
3268         my ($ds, $drive) = @_;
3269         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3270     });
3271
3272     foreach my $snapname (keys %{$conf->{snapshots}}) {
3273         my $snap = $conf->{snapshots}->{$snapname};
3274         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3275         foreach_drive($snap, sub {
3276             my ($ds, $drive) = @_;
3277             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3278         });
3279     }
3280
3281     foreach my $volid (keys %$volhash) {
3282         &$func($volid, $volhash->{$volid}, @param);
3283     }
3284 }
3285
3286 sub conf_has_serial {
3287     my ($conf) = @_;
3288
3289     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3290         if ($conf->{"serial$i"}) {
3291             return 1;
3292         }
3293     }
3294
3295     return 0;
3296 }
3297
3298 sub vga_conf_has_spice {
3299     my ($vga) = @_;
3300
3301     my $vgaconf = parse_vga($vga);
3302     my $vgatype = $vgaconf->{type};
3303     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3304
3305     return $1 || 1;
3306 }
3307
3308 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3309 sub get_host_arch() {
3310     $host_arch = (POSIX::uname())[4] if !$host_arch;
3311     return $host_arch;
3312 }
3313
3314 sub is_native($) {
3315     my ($arch) = @_;
3316     return get_host_arch() eq $arch;
3317 }
3318
3319 my $default_machines = {
3320     x86_64 => 'pc',
3321     aarch64 => 'virt',
3322 };
3323
3324 sub get_basic_machine_info {
3325     my ($conf, $forcemachine) = @_;
3326
3327     my $arch = $conf->{arch} // get_host_arch();
3328     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3329     return ($arch, $machine);
3330 }
3331
3332 sub get_ovmf_files($) {
3333     my ($arch) = @_;
3334
3335     my $ovmf = $OVMF->{$arch}
3336         or die "no OVMF images known for architecture '$arch'\n";
3337
3338     return @$ovmf;
3339 }
3340
3341 my $Arch2Qemu = {
3342     aarch64 => '/usr/bin/qemu-system-aarch64',
3343     x86_64 => '/usr/bin/qemu-system-x86_64',
3344 };
3345 sub get_command_for_arch($) {
3346     my ($arch) = @_;
3347     return '/usr/bin/kvm' if is_native($arch);
3348
3349     my $cmd = $Arch2Qemu->{$arch}
3350         or die "don't know how to emulate architecture '$arch'\n";
3351     return $cmd;
3352 }
3353
3354 sub get_cpu_options {
3355     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3356
3357     my $cpuFlags = [];
3358     my $ostype = $conf->{ostype};
3359
3360     my $cpu = $kvm ? "kvm64" : "qemu64";
3361     if ($arch eq 'aarch64') {
3362         $cpu = 'cortex-a57';
3363     }
3364     my $hv_vendor_id;
3365     if (my $cputype = $conf->{cpu}) {
3366         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3367             or die "Cannot parse cpu description: $cputype\n";
3368         $cpu = $cpuconf->{cputype};
3369         $kvm_off = 1 if $cpuconf->{hidden};
3370         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3371
3372         if (defined(my $flags = $cpuconf->{flags})) {
3373             push @$cpuFlags, split(";", $flags);
3374         }
3375     }
3376
3377     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3378
3379     push @$cpuFlags , '-x2apic'
3380         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3381
3382     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3383
3384     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3385
3386     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3387
3388         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3389         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3390     }
3391
3392     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3393
3394     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3395
3396     push @$cpuFlags, 'kvm=off' if $kvm_off;
3397
3398     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3399         push @$cpuFlags, "vendor=${cpu_vendor}"
3400             if $cpu_vendor ne 'default';
3401     } elsif ($arch ne 'aarch64') {
3402         die "internal error"; # should not happen
3403     }
3404
3405     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3406
3407     return ('-cpu', $cpu);
3408 }
3409
3410 sub config_to_command {
3411     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3412
3413     my $cmd = [];
3414     my $globalFlags = [];
3415     my $machineFlags = [];
3416     my $rtcFlags = [];
3417     my $devices = [];
3418     my $pciaddr = '';
3419     my $bridges = {};
3420     my $kvmver = kvm_user_version();
3421     my $vernum = 0; # unknown
3422     my $ostype = $conf->{ostype};
3423     my $winversion = windows_version($ostype);
3424     my $kvm = $conf->{kvm};
3425
3426     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3427     $kvm //= 1 if is_native($arch);
3428
3429     if ($kvm) {
3430         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3431             if !defined kvm_version();
3432     }
3433
3434     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3435         $vernum = $1*1000000+$2*1000;
3436     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3437         $vernum = $1*1000000+$2*1000+$3;
3438     }
3439
3440     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3441
3442     my $have_ovz = -f '/proc/vz/vestat';
3443
3444     my $q35 = machine_type_is_q35($conf);
3445     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3446     my $use_old_bios_files = undef;
3447     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3448
3449     my $cpuunits = defined($conf->{cpuunits}) ?
3450             $conf->{cpuunits} : $defaults->{cpuunits};
3451
3452     push @$cmd, get_command_for_arch($arch);
3453
3454     push @$cmd, '-id', $vmid;
3455
3456     my $vmname = $conf->{name} || "vm$vmid";
3457
3458     push @$cmd, '-name', $vmname;
3459
3460     my $use_virtio = 0;
3461
3462     my $qmpsocket = qmp_socket($vmid);
3463     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3464     push @$cmd, '-mon', "chardev=qmp,mode=control";
3465
3466     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3467         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3468         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3469     }
3470
3471     push @$cmd, '-pidfile' , pidfile_name($vmid);
3472
3473     push @$cmd, '-daemonize';
3474
3475     if ($conf->{smbios1}) {
3476         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3477     }
3478
3479     if ($conf->{vmgenid}) {
3480         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3481     }
3482
3483     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3484     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3485         die "uefi base image not found\n" if ! -f $ovmf_code;
3486
3487         my $path;
3488         my $format;
3489         if (my $efidisk = $conf->{efidisk0}) {
3490             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3491             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3492             $format = $d->{format};
3493             if ($storeid) {
3494                 $path = PVE::Storage::path($storecfg, $d->{file});
3495                 if (!defined($format)) {
3496                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3497                     $format = qemu_img_format($scfg, $volname);
3498                 }
3499             } else {
3500                 $path = $d->{file};
3501                 die "efidisk format must be specified\n"
3502                     if !defined($format);
3503             }
3504         } else {
3505             warn "no efidisk configured! Using temporary efivars disk.\n";
3506             $path = "/tmp/$vmid-ovmf.fd";
3507             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3508             $format = 'raw';
3509         }
3510
3511         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3512         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3513     }
3514
3515
3516     # add usb controllers
3517     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3518     push @$devices, @usbcontrollers if @usbcontrollers;
3519     my $vga = parse_vga($conf->{vga});
3520
3521     my $qxlnum = vga_conf_has_spice($conf->{vga});
3522     $vga->{type} = 'qxl' if $qxlnum;
3523
3524     if (!$vga->{type}) {
3525         if ($arch eq 'aarch64') {
3526             $vga->{type} = 'virtio';
3527         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3528             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3529         } else {
3530             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3531         }
3532     }
3533
3534     # enable absolute mouse coordinates (needed by vnc)
3535     my $tablet;
3536     if (defined($conf->{tablet})) {
3537         $tablet = $conf->{tablet};
3538     } else {
3539         $tablet = $defaults->{tablet};
3540         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3541         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3542     }
3543
3544     if ($tablet) {
3545         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3546         my $kbd = print_keyboarddevice_full($conf, $arch);
3547         push @$devices, '-device', $kbd if defined($kbd);
3548     }
3549
3550     my $kvm_off = 0;
3551     my $gpu_passthrough;
3552
3553     # host pci devices
3554     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3555         my $d = parse_hostpci($conf->{"hostpci$i"});
3556         next if !$d;
3557
3558         my $pcie = $d->{pcie};
3559         if($pcie){
3560             die "q35 machine model is not enabled" if !$q35;
3561             # win7 wants to have the pcie devices directly on the pcie bus
3562             # instead of in the root port
3563             if ($winversion == 7) {
3564                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3565             } else {
3566                 $pciaddr = print_pcie_addr("hostpci$i");
3567             }
3568         }else{
3569             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3570         }
3571
3572         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3573         my $romfile = $d->{romfile};
3574
3575         my $xvga = '';
3576         if ($d->{'x-vga'}) {
3577             $xvga = ',x-vga=on';
3578             $kvm_off = 1;
3579             $vga->{type} = 'none' if !defined($conf->{vga});
3580             $gpu_passthrough = 1;
3581
3582             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3583                 $xvga = "";
3584             }
3585         }
3586         my $pcidevices = $d->{pciid};
3587         my $multifunction = 1 if @$pcidevices > 1;
3588         my $sysfspath;
3589         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3590             my $id = $pcidevices->[0]->{id};
3591             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3592             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3593         } elsif ($d->{mdev}) {
3594             warn "ignoring mediated device with multifunction device\n";
3595         }
3596
3597         my $j=0;
3598         foreach my $pcidevice (@$pcidevices) {
3599
3600             my $id = "hostpci$i";
3601             $id .= ".$j" if $multifunction;
3602             my $addr = $pciaddr;
3603             $addr .= ".$j" if $multifunction;
3604             my $devicestr = "vfio-pci";
3605             if ($sysfspath) {
3606                 $devicestr .= ",sysfsdev=$sysfspath";
3607             } else {
3608                 $devicestr .= ",host=$pcidevice->{id}";
3609             }
3610             $devicestr .= ",id=$id$addr";
3611
3612             if($j == 0){
3613                 $devicestr .= "$rombar$xvga";
3614                 $devicestr .= ",multifunction=on" if $multifunction;
3615                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3616             }
3617
3618             push @$devices, '-device', $devicestr;
3619             $j++;
3620         }
3621     }
3622
3623     # usb devices
3624     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3625     push @$devices, @usbdevices if @usbdevices;
3626     # serial devices
3627     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3628         if (my $path = $conf->{"serial$i"}) {
3629             if ($path eq 'socket') {
3630                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3631                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3632                 # On aarch64, serial0 is the UART device. Qemu only allows
3633                 # connecting UART devices via the '-serial' command line, as
3634                 # the device has a fixed slot on the hardware...
3635                 if ($arch eq 'aarch64' && $i == 0) {
3636                     push @$devices, '-serial', "chardev:serial$i";
3637                 } else {
3638                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3639                 }
3640             } else {
3641                 die "no such serial device\n" if ! -c $path;
3642                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3643                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3644             }
3645         }
3646     }
3647
3648     # parallel devices
3649     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3650         if (my $path = $conf->{"parallel$i"}) {
3651             die "no such parallel device\n" if ! -c $path;
3652             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3653             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3654             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3655         }
3656     }
3657
3658
3659     my $sockets = 1;
3660     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3661     $sockets = $conf->{sockets} if  $conf->{sockets};
3662
3663     my $cores = $conf->{cores} || 1;
3664
3665     my $maxcpus = $sockets * $cores;
3666
3667     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3668
3669     my $allowed_vcpus = $cpuinfo->{cpus};
3670
3671     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3672         if ($allowed_vcpus < $maxcpus);
3673
3674     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3675
3676         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3677         for (my $i = 2; $i <= $vcpus; $i++)  {
3678             my $cpustr = print_cpu_device($conf,$i);
3679             push @$cmd, '-device', $cpustr;
3680         }
3681
3682     } else {
3683
3684         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3685     }
3686     push @$cmd, '-nodefaults';
3687
3688     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3689
3690     my $bootindex_hash = {};
3691     my $i = 1;
3692     foreach my $o (split(//, $bootorder)) {
3693         $bootindex_hash->{$o} = $i*100;
3694         $i++;
3695     }
3696
3697     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3698
3699     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3700
3701     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3702
3703     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3704         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3705         my $socket = vnc_socket($vmid);
3706         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3707     } else {
3708         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3709         push @$cmd, '-nographic';
3710     }
3711
3712     # time drift fix
3713     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3714
3715     my $useLocaltime = $conf->{localtime};
3716
3717     if ($winversion >= 5) { # windows
3718         $useLocaltime = 1 if !defined($conf->{localtime});
3719
3720         # use time drift fix when acpi is enabled
3721         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3722             $tdf = 1 if !defined($conf->{tdf});
3723         }
3724     }
3725
3726     if ($winversion >= 6) {
3727         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3728         push @$cmd, '-no-hpet';
3729     }
3730
3731     push @$rtcFlags, 'driftfix=slew' if $tdf;
3732
3733     if (!$kvm) {
3734         push @$machineFlags, 'accel=tcg';
3735     }
3736
3737     if ($machine_type) {
3738         push @$machineFlags, "type=${machine_type}";
3739     }
3740
3741     if ($conf->{startdate}) {
3742         push @$rtcFlags, "base=$conf->{startdate}";
3743     } elsif ($useLocaltime) {
3744         push @$rtcFlags, 'base=localtime';
3745     }
3746
3747     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3748
3749     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3750
3751     push @$cmd, '-S' if $conf->{freeze};
3752
3753     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3754
3755     # enable sound
3756     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3757     #push @$cmd, '-soundhw', 'es1370';
3758     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3759
3760     if (parse_guest_agent($conf)->{enabled}) {
3761         my $qgasocket = qmp_socket($vmid, 1);
3762         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3763         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3764         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3765         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3766     }
3767
3768     my $spice_port;
3769
3770     if ($qxlnum) {
3771         if ($qxlnum > 1) {
3772             if ($winversion){
3773                 for(my $i = 1; $i < $qxlnum; $i++){
3774                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3775                 }
3776             } else {
3777                 # assume other OS works like Linux
3778                 my ($ram, $vram) = ("134217728", "67108864");
3779                 if ($vga->{memory}) {
3780                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3781                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3782                 }
3783                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3784                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3785             }
3786         }
3787
3788         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3789
3790         my $nodename = PVE::INotify::nodename();
3791         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3792         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3793         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3794         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3795         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3796
3797         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3798
3799         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3800         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3801         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3802     }
3803
3804     # enable balloon by default, unless explicitly disabled
3805     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3806         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3807         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3808     }
3809
3810     if ($conf->{watchdog}) {
3811         my $wdopts = parse_watchdog($conf->{watchdog});
3812         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3813         my $watchdog = $wdopts->{model} || 'i6300esb';
3814         push @$devices, '-device', "$watchdog$pciaddr";
3815         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3816     }
3817
3818     my $vollist = [];
3819     my $scsicontroller = {};
3820     my $ahcicontroller = {};
3821     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3822
3823     # Add iscsi initiator name if available
3824     if (my $initiator = get_initiator_name()) {
3825         push @$devices, '-iscsi', "initiator-name=$initiator";
3826     }
3827
3828     foreach_drive($conf, sub {
3829         my ($ds, $drive) = @_;
3830
3831         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3832             push @$vollist, $drive->{file};
3833         }
3834
3835         # ignore efidisk here, already added in bios/fw handling code above
3836         return if $drive->{interface} eq 'efidisk';
3837
3838         $use_virtio = 1 if $ds =~ m/^virtio/;
3839
3840         if (drive_is_cdrom ($drive)) {
3841             if ($bootindex_hash->{d}) {
3842                 $drive->{bootindex} = $bootindex_hash->{d};
3843                 $bootindex_hash->{d} += 1;
3844             }
3845         } else {
3846             if ($bootindex_hash->{c}) {
3847                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3848                 $bootindex_hash->{c} += 1;
3849             }
3850         }
3851
3852         if($drive->{interface} eq 'virtio'){
3853            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3854         }
3855
3856         if ($drive->{interface} eq 'scsi') {
3857
3858             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3859
3860             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3861             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3862
3863             my $iothread = '';
3864             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3865                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3866                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3867             } elsif ($drive->{iothread}) {
3868                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3869             }
3870
3871             my $queues = '';
3872             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3873                 $queues = ",num_queues=$drive->{queues}";
3874             }
3875
3876             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3877             $scsicontroller->{$controller}=1;
3878         }
3879
3880         if ($drive->{interface} eq 'sata') {
3881            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3882            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3883            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3884            $ahcicontroller->{$controller}=1;
3885         }
3886
3887         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3888         push @$devices, '-drive',$drive_cmd;
3889         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3890     });
3891
3892     for (my $i = 0; $i < $MAX_NETS; $i++) {
3893          next if !$conf->{"net$i"};
3894          my $d = parse_net($conf->{"net$i"});
3895          next if !$d;
3896
3897          $use_virtio = 1 if $d->{model} eq 'virtio';
3898
3899          if ($bootindex_hash->{n}) {
3900             $d->{bootindex} = $bootindex_hash->{n};
3901             $bootindex_hash->{n} += 1;
3902          }
3903
3904          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3905          push @$devices, '-netdev', $netdevfull;
3906
3907          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3908          push @$devices, '-device', $netdevicefull;
3909     }
3910
3911     if (!$q35) {
3912         # add pci bridges
3913         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3914            $bridges->{1} = 1;
3915            $bridges->{2} = 1;
3916         }
3917
3918         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3919
3920         while (my ($k, $v) = each %$bridges) {
3921             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3922             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3923         }
3924     }
3925
3926     push @$cmd, @$devices;
3927     push @$cmd, '-rtc', join(',', @$rtcFlags)
3928         if scalar(@$rtcFlags);
3929     push @$cmd, '-machine', join(',', @$machineFlags)
3930         if scalar(@$machineFlags);
3931     push @$cmd, '-global', join(',', @$globalFlags)
3932         if scalar(@$globalFlags);
3933
3934     # add custom args
3935     if ($conf->{args}) {
3936         my $aa = PVE::Tools::split_args($conf->{args});
3937         push @$cmd, @$aa;
3938     }
3939
3940     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3941 }
3942
3943 sub vnc_socket {
3944     my ($vmid) = @_;
3945     return "${var_run_tmpdir}/$vmid.vnc";
3946 }
3947
3948 sub spice_port {
3949     my ($vmid) = @_;
3950
3951     my $res = vm_mon_cmd($vmid, 'query-spice');
3952
3953     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3954 }
3955
3956 sub qmp_socket {
3957     my ($vmid, $qga, $name) = @_;
3958     my $sockettype = $qga ? 'qga' : 'qmp';
3959     my $ext = $name ? '-'.$name : '';
3960     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3961 }
3962
3963 sub pidfile_name {
3964     my ($vmid) = @_;
3965     return "${var_run_tmpdir}/$vmid.pid";
3966 }
3967
3968 sub vm_devices_list {
3969     my ($vmid) = @_;
3970
3971     my $res = vm_mon_cmd($vmid, 'query-pci');
3972     my $devices_to_check = [];
3973     my $devices = {};
3974     foreach my $pcibus (@$res) {
3975         push @$devices_to_check, @{$pcibus->{devices}},
3976     }
3977
3978     while (@$devices_to_check) {
3979         my $to_check = [];
3980         for my $d (@$devices_to_check) {
3981             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3982             next if !$d->{'pci_bridge'};
3983
3984             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3985             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3986         }
3987         $devices_to_check = $to_check;
3988     }
3989
3990     my $resblock = vm_mon_cmd($vmid, 'query-block');
3991     foreach my $block (@$resblock) {
3992         if($block->{device} =~ m/^drive-(\S+)/){
3993                 $devices->{$1} = 1;
3994         }
3995     }
3996
3997     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3998     foreach my $mice (@$resmice) {
3999         if ($mice->{name} eq 'QEMU HID Tablet') {
4000             $devices->{tablet} = 1;
4001             last;
4002         }
4003     }
4004
4005     # for usb devices there is no query-usb
4006     # but we can iterate over the entries in
4007     # qom-list path=/machine/peripheral
4008     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4009     foreach my $per (@$resperipheral) {
4010         if ($per->{name} =~ m/^usb\d+$/) {
4011             $devices->{$per->{name}} = 1;
4012         }
4013     }
4014
4015     return $devices;
4016 }
4017
4018 sub vm_deviceplug {
4019     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4020
4021     my $q35 = machine_type_is_q35($conf);
4022
4023     my $devices_list = vm_devices_list($vmid);
4024     return 1 if defined($devices_list->{$deviceid});
4025
4026     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4027
4028     if ($deviceid eq 'tablet') {
4029
4030         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4031
4032     } elsif ($deviceid eq 'keyboard') {
4033
4034         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4035
4036     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4037
4038         die "usb hotplug currently not reliable\n";
4039         # since we can't reliably hot unplug all added usb devices
4040         # and usb passthrough disables live migration
4041         # we disable usb hotplugging for now
4042         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4043
4044     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4045
4046         qemu_iothread_add($vmid, $deviceid, $device);
4047
4048         qemu_driveadd($storecfg, $vmid, $device);
4049         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4050
4051         qemu_deviceadd($vmid, $devicefull);
4052         eval { qemu_deviceaddverify($vmid, $deviceid); };
4053         if (my $err = $@) {
4054             eval { qemu_drivedel($vmid, $deviceid); };
4055             warn $@ if $@;
4056             die $err;
4057         }
4058
4059     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4060
4061
4062         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4063         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4064         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4065
4066         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4067
4068         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4069             qemu_iothread_add($vmid, $deviceid, $device);
4070             $devicefull .= ",iothread=iothread-$deviceid";
4071         }
4072
4073         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4074             $devicefull .= ",num_queues=$device->{queues}";
4075         }
4076
4077         qemu_deviceadd($vmid, $devicefull);
4078         qemu_deviceaddverify($vmid, $deviceid);
4079
4080     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4081
4082         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4083         qemu_driveadd($storecfg, $vmid, $device);
4084
4085         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4086         eval { qemu_deviceadd($vmid, $devicefull); };
4087         if (my $err = $@) {
4088             eval { qemu_drivedel($vmid, $deviceid); };
4089             warn $@ if $@;
4090             die $err;
4091         }
4092
4093     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4094
4095         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4096
4097         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4098         my $use_old_bios_files = undef;
4099         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4100
4101         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4102         qemu_deviceadd($vmid, $netdevicefull);
4103         eval { qemu_deviceaddverify($vmid, $deviceid); };
4104         if (my $err = $@) {
4105             eval { qemu_netdevdel($vmid, $deviceid); };
4106             warn $@ if $@;
4107             die $err;
4108         }
4109
4110     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4111
4112         my $bridgeid = $2;
4113         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4114         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4115
4116         qemu_deviceadd($vmid, $devicefull);
4117         qemu_deviceaddverify($vmid, $deviceid);
4118
4119     } else {
4120         die "can't hotplug device '$deviceid'\n";
4121     }
4122
4123     return 1;
4124 }
4125
4126 # fixme: this should raise exceptions on error!
4127 sub vm_deviceunplug {
4128     my ($vmid, $conf, $deviceid) = @_;
4129
4130     my $devices_list = vm_devices_list($vmid);
4131     return 1 if !defined($devices_list->{$deviceid});
4132
4133     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4134
4135     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4136
4137         qemu_devicedel($vmid, $deviceid);
4138
4139     } elsif ($deviceid =~ m/^usb\d+$/) {
4140
4141         die "usb hotplug currently not reliable\n";
4142         # when unplugging usb devices this way,
4143         # there may be remaining usb controllers/hubs
4144         # so we disable it for now
4145         qemu_devicedel($vmid, $deviceid);
4146         qemu_devicedelverify($vmid, $deviceid);
4147
4148     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4149
4150         qemu_devicedel($vmid, $deviceid);
4151         qemu_devicedelverify($vmid, $deviceid);
4152         qemu_drivedel($vmid, $deviceid);
4153         qemu_iothread_del($conf, $vmid, $deviceid);
4154
4155     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4156
4157         qemu_devicedel($vmid, $deviceid);
4158         qemu_devicedelverify($vmid, $deviceid);
4159         qemu_iothread_del($conf, $vmid, $deviceid);
4160
4161     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4162
4163         qemu_devicedel($vmid, $deviceid);
4164         qemu_drivedel($vmid, $deviceid);
4165         qemu_deletescsihw($conf, $vmid, $deviceid);
4166
4167     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4168
4169         qemu_devicedel($vmid, $deviceid);
4170         qemu_devicedelverify($vmid, $deviceid);
4171         qemu_netdevdel($vmid, $deviceid);
4172
4173     } else {
4174         die "can't unplug device '$deviceid'\n";
4175     }
4176
4177     return 1;
4178 }
4179
4180 sub qemu_deviceadd {
4181     my ($vmid, $devicefull) = @_;
4182
4183     $devicefull = "driver=".$devicefull;
4184     my %options =  split(/[=,]/, $devicefull);
4185
4186     vm_mon_cmd($vmid, "device_add" , %options);
4187 }
4188
4189 sub qemu_devicedel {
4190     my ($vmid, $deviceid) = @_;
4191
4192     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4193 }
4194
4195 sub qemu_iothread_add {
4196     my($vmid, $deviceid, $device) = @_;
4197
4198     if ($device->{iothread}) {
4199         my $iothreads = vm_iothreads_list($vmid);
4200         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4201     }
4202 }
4203
4204 sub qemu_iothread_del {
4205     my($conf, $vmid, $deviceid) = @_;
4206
4207     my $device = parse_drive($deviceid, $conf->{$deviceid});
4208     if ($device->{iothread}) {
4209         my $iothreads = vm_iothreads_list($vmid);
4210         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4211     }
4212 }
4213
4214 sub qemu_objectadd {
4215     my($vmid, $objectid, $qomtype) = @_;
4216
4217     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4218
4219     return 1;
4220 }
4221
4222 sub qemu_objectdel {
4223     my($vmid, $objectid) = @_;
4224
4225     vm_mon_cmd($vmid, "object-del", id => $objectid);
4226
4227     return 1;
4228 }
4229
4230 sub qemu_driveadd {
4231     my ($storecfg, $vmid, $device) = @_;
4232
4233     my $drive = print_drive_full($storecfg, $vmid, $device);
4234     $drive =~ s/\\/\\\\/g;
4235     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4236
4237     # If the command succeeds qemu prints: "OK"
4238     return 1 if $ret =~ m/OK/s;
4239
4240     die "adding drive failed: $ret\n";
4241 }
4242
4243 sub qemu_drivedel {
4244     my($vmid, $deviceid) = @_;
4245
4246     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4247     $ret =~ s/^\s+//;
4248
4249     return 1 if $ret eq "";
4250
4251     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4252     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4253
4254     die "deleting drive $deviceid failed : $ret\n";
4255 }
4256
4257 sub qemu_deviceaddverify {
4258     my ($vmid, $deviceid) = @_;
4259
4260     for (my $i = 0; $i <= 5; $i++) {
4261          my $devices_list = vm_devices_list($vmid);
4262          return 1 if defined($devices_list->{$deviceid});
4263          sleep 1;
4264     }
4265
4266     die "error on hotplug device '$deviceid'\n";
4267 }
4268
4269
4270 sub qemu_devicedelverify {
4271     my ($vmid, $deviceid) = @_;
4272
4273     # need to verify that the device is correctly removed as device_del
4274     # is async and empty return is not reliable
4275
4276     for (my $i = 0; $i <= 5; $i++) {
4277          my $devices_list = vm_devices_list($vmid);
4278          return 1 if !defined($devices_list->{$deviceid});
4279          sleep 1;
4280     }
4281
4282     die "error on hot-unplugging device '$deviceid'\n";
4283 }
4284
4285 sub qemu_findorcreatescsihw {
4286     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4287
4288     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4289
4290     my $scsihwid="$controller_prefix$controller";
4291     my $devices_list = vm_devices_list($vmid);
4292
4293     if(!defined($devices_list->{$scsihwid})) {
4294         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4295     }
4296
4297     return 1;
4298 }
4299
4300 sub qemu_deletescsihw {
4301     my ($conf, $vmid, $opt) = @_;
4302
4303     my $device = parse_drive($opt, $conf->{$opt});
4304
4305     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4306         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4307         return 1;
4308     }
4309
4310     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4311
4312     my $devices_list = vm_devices_list($vmid);
4313     foreach my $opt (keys %{$devices_list}) {
4314         if (PVE::QemuServer::is_valid_drivename($opt)) {
4315             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4316             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4317                 return 1;
4318             }
4319         }
4320     }
4321
4322     my $scsihwid="scsihw$controller";
4323
4324     vm_deviceunplug($vmid, $conf, $scsihwid);
4325
4326     return 1;
4327 }
4328
4329 sub qemu_add_pci_bridge {
4330     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4331
4332     my $bridges = {};
4333
4334     my $bridgeid;
4335
4336     print_pci_addr($device, $bridges, $arch, $machine_type);
4337
4338     while (my ($k, $v) = each %$bridges) {
4339         $bridgeid = $k;
4340     }
4341     return 1 if !defined($bridgeid) || $bridgeid < 1;
4342
4343     my $bridge = "pci.$bridgeid";
4344     my $devices_list = vm_devices_list($vmid);
4345
4346     if (!defined($devices_list->{$bridge})) {
4347         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4348     }
4349
4350     return 1;
4351 }
4352
4353 sub qemu_set_link_status {
4354     my ($vmid, $device, $up) = @_;
4355
4356     vm_mon_cmd($vmid, "set_link", name => $device,
4357                up => $up ? JSON::true : JSON::false);
4358 }
4359
4360 sub qemu_netdevadd {
4361     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4362
4363     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4364     my %options =  split(/[=,]/, $netdev);
4365
4366     vm_mon_cmd($vmid, "netdev_add",  %options);
4367     return 1;
4368 }
4369
4370 sub qemu_netdevdel {
4371     my ($vmid, $deviceid) = @_;
4372
4373     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4374 }
4375
4376 sub qemu_usb_hotplug {
4377     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4378
4379     return if !$device;
4380
4381     # remove the old one first
4382     vm_deviceunplug($vmid, $conf, $deviceid);
4383
4384     # check if xhci controller is necessary and available
4385     if ($device->{usb3}) {
4386
4387         my $devicelist = vm_devices_list($vmid);
4388
4389         if (!$devicelist->{xhci}) {
4390             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4391             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4392         }
4393     }
4394     my $d = parse_usb_device($device->{host});
4395     $d->{usb3} = $device->{usb3};
4396
4397     # add the new one
4398     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4399 }
4400
4401 sub qemu_cpu_hotplug {
4402     my ($vmid, $conf, $vcpus) = @_;
4403
4404     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4405
4406     my $sockets = 1;
4407     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4408     $sockets = $conf->{sockets} if  $conf->{sockets};
4409     my $cores = $conf->{cores} || 1;
4410     my $maxcpus = $sockets * $cores;
4411
4412     $vcpus = $maxcpus if !$vcpus;
4413
4414     die "you can't add more vcpus than maxcpus\n"
4415         if $vcpus > $maxcpus;
4416
4417     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4418
4419     if ($vcpus < $currentvcpus) {
4420
4421         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4422
4423             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4424                 qemu_devicedel($vmid, "cpu$i");
4425                 my $retry = 0;
4426                 my $currentrunningvcpus = undef;
4427                 while (1) {
4428                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4429                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4430                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4431                     $retry++;
4432                     sleep 1;
4433                 }
4434                 #update conf after each succesfull cpu unplug
4435                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4436                 PVE::QemuConfig->write_config($vmid, $conf);
4437             }
4438         } else {
4439             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4440         }
4441
4442         return;
4443     }
4444
4445     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4446     die "vcpus in running vm does not match its configuration\n"
4447         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4448
4449     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4450
4451         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4452             my $cpustr = print_cpu_device($conf, $i);
4453             qemu_deviceadd($vmid, $cpustr);
4454
4455             my $retry = 0;
4456             my $currentrunningvcpus = undef;
4457             while (1) {
4458                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4459                 last if scalar(@{$currentrunningvcpus}) == $i;
4460                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4461                 sleep 1;
4462                 $retry++;
4463             }
4464             #update conf after each succesfull cpu hotplug
4465             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4466             PVE::QemuConfig->write_config($vmid, $conf);
4467         }
4468     } else {
4469
4470         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4471             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4472         }
4473     }
4474 }
4475
4476 sub qemu_block_set_io_throttle {
4477     my ($vmid, $deviceid,
4478         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4479         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4480         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4481         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4482
4483     return if !check_running($vmid) ;
4484
4485     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4486         bps => int($bps),
4487         bps_rd => int($bps_rd),
4488         bps_wr => int($bps_wr),
4489         iops => int($iops),
4490         iops_rd => int($iops_rd),
4491         iops_wr => int($iops_wr),
4492         bps_max => int($bps_max),
4493         bps_rd_max => int($bps_rd_max),
4494         bps_wr_max => int($bps_wr_max),
4495         iops_max => int($iops_max),
4496         iops_rd_max => int($iops_rd_max),
4497         iops_wr_max => int($iops_wr_max),
4498         bps_max_length => int($bps_max_length),
4499         bps_rd_max_length => int($bps_rd_max_length),
4500         bps_wr_max_length => int($bps_wr_max_length),
4501         iops_max_length => int($iops_max_length),
4502         iops_rd_max_length => int($iops_rd_max_length),
4503         iops_wr_max_length => int($iops_wr_max_length),
4504     );
4505
4506 }
4507
4508 # old code, only used to shutdown old VM after update
4509 sub __read_avail {
4510     my ($fh, $timeout) = @_;
4511
4512     my $sel = new IO::Select;
4513     $sel->add($fh);
4514
4515     my $res = '';
4516     my $buf;
4517
4518     my @ready;
4519     while (scalar (@ready = $sel->can_read($timeout))) {
4520         my $count;
4521         if ($count = $fh->sysread($buf, 8192)) {
4522             if ($buf =~ /^(.*)\(qemu\) $/s) {
4523                 $res .= $1;
4524                 last;
4525             } else {
4526                 $res .= $buf;
4527             }
4528         } else {
4529             if (!defined($count)) {
4530                 die "$!\n";
4531             }
4532             last;
4533         }
4534     }
4535
4536     die "monitor read timeout\n" if !scalar(@ready);
4537
4538     return $res;
4539 }
4540
4541 sub qemu_block_resize {
4542     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4543
4544     my $running = check_running($vmid);
4545
4546     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4547
4548     return if !$running;
4549
4550     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4551
4552 }
4553
4554 sub qemu_volume_snapshot {
4555     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4556
4557     my $running = check_running($vmid);
4558
4559     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4560         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4561     } else {
4562         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4563     }
4564 }
4565
4566 sub qemu_volume_snapshot_delete {
4567     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4568
4569     my $running = check_running($vmid);
4570
4571     if($running) {
4572
4573         $running = undef;
4574         my $conf = PVE::QemuConfig->load_config($vmid);
4575         foreach_drive($conf, sub {
4576             my ($ds, $drive) = @_;
4577             $running = 1 if $drive->{file} eq $volid;
4578         });
4579     }
4580
4581     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4582         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4583     } else {
4584         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4585     }
4586 }
4587
4588 sub set_migration_caps {
4589     my ($vmid) = @_;
4590
4591     my $cap_ref = [];
4592
4593     my $enabled_cap = {
4594         "auto-converge" => 1,
4595         "xbzrle" => 1,
4596         "x-rdma-pin-all" => 0,
4597         "zero-blocks" => 0,
4598         "compress" => 0
4599     };
4600
4601     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4602
4603     for my $supported_capability (@$supported_capabilities) {
4604         push @$cap_ref, {
4605             capability => $supported_capability->{capability},
4606             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4607         };
4608     }
4609
4610     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4611 }
4612
4613 my $fast_plug_option = {
4614     'lock' => 1,
4615     'name' => 1,
4616     'onboot' => 1,
4617     'shares' => 1,
4618     'startup' => 1,
4619     'description' => 1,
4620     'protection' => 1,
4621     'vmstatestorage' => 1,
4622     'hookscript' => 1,
4623 };
4624
4625 # hotplug changes in [PENDING]
4626 # $selection hash can be used to only apply specified options, for
4627 # example: { cores => 1 } (only apply changed 'cores')
4628 # $errors ref is used to return error messages
4629 sub vmconfig_hotplug_pending {
4630     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4631
4632     my $defaults = load_defaults();
4633     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4634
4635     # commit values which do not have any impact on running VM first
4636     # Note: those option cannot raise errors, we we do not care about
4637     # $selection and always apply them.
4638
4639     my $add_error = sub {
4640         my ($opt, $msg) = @_;
4641         $errors->{$opt} = "hotplug problem - $msg";
4642     };
4643
4644     my $changes = 0;
4645     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4646         if ($fast_plug_option->{$opt}) {
4647             $conf->{$opt} = $conf->{pending}->{$opt};
4648             delete $conf->{pending}->{$opt};
4649             $changes = 1;
4650         }
4651     }
4652
4653     if ($changes) {
4654         PVE::QemuConfig->write_config($vmid, $conf);
4655         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4656     }
4657
4658     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4659
4660     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4661     while (my ($opt, $force) = each %$pending_delete_hash) {
4662         next if $selection && !$selection->{$opt};
4663         eval {
4664             if ($opt eq 'hotplug') {
4665                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4666             } elsif ($opt eq 'tablet') {
4667                 die "skip\n" if !$hotplug_features->{usb};
4668                 if ($defaults->{tablet}) {
4669                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4670                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4671                         if $arch eq 'aarch64';
4672                 } else {
4673                     vm_deviceunplug($vmid, $conf, 'tablet');
4674                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4675                 }
4676             } elsif ($opt =~ m/^usb\d+/) {
4677                 die "skip\n";
4678                 # since we cannot reliably hot unplug usb devices
4679                 # we are disabling it
4680                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4681                 vm_deviceunplug($vmid, $conf, $opt);
4682             } elsif ($opt eq 'vcpus') {
4683                 die "skip\n" if !$hotplug_features->{cpu};
4684                 qemu_cpu_hotplug($vmid, $conf, undef);
4685             } elsif ($opt eq 'balloon') {
4686                 # enable balloon device is not hotpluggable
4687                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4688                 # here we reset the ballooning value to memory
4689                 my $balloon = $conf->{memory} || $defaults->{memory};
4690                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4691             } elsif ($fast_plug_option->{$opt}) {
4692                 # do nothing
4693             } elsif ($opt =~ m/^net(\d+)$/) {
4694                 die "skip\n" if !$hotplug_features->{network};
4695                 vm_deviceunplug($vmid, $conf, $opt);
4696             } elsif (is_valid_drivename($opt)) {
4697                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4698                 vm_deviceunplug($vmid, $conf, $opt);
4699                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4700             } elsif ($opt =~ m/^memory$/) {
4701                 die "skip\n" if !$hotplug_features->{memory};
4702                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4703             } elsif ($opt eq 'cpuunits') {
4704                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4705             } elsif ($opt eq 'cpulimit') {
4706                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4707             } else {
4708                 die "skip\n";
4709             }
4710         };
4711         if (my $err = $@) {
4712             &$add_error($opt, $err) if $err ne "skip\n";
4713         } else {
4714             # save new config if hotplug was successful
4715             delete $conf->{$opt};
4716             vmconfig_undelete_pending_option($conf, $opt);
4717             PVE::QemuConfig->write_config($vmid, $conf);
4718             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4719         }
4720     }
4721
4722     my $apply_pending_cloudinit;
4723     $apply_pending_cloudinit = sub {
4724         my ($key, $value) = @_;
4725         $apply_pending_cloudinit = sub {}; # once is enough
4726
4727         my @cloudinit_opts = keys %$confdesc_cloudinit;
4728         foreach my $opt (keys %{$conf->{pending}}) {
4729             next if !grep { $_ eq $opt } @cloudinit_opts;
4730             $conf->{$opt} = delete $conf->{pending}->{$opt};
4731         }
4732
4733         my $new_conf = { %$conf };
4734         $new_conf->{$key} = $value;
4735         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4736     };
4737
4738     foreach my $opt (keys %{$conf->{pending}}) {
4739         next if $selection && !$selection->{$opt};
4740         my $value = $conf->{pending}->{$opt};
4741         eval {
4742             if ($opt eq 'hotplug') {
4743                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4744             } elsif ($opt eq 'tablet') {
4745                 die "skip\n" if !$hotplug_features->{usb};
4746                 if ($value == 1) {
4747                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4748                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4749                         if $arch eq 'aarch64';
4750                 } elsif ($value == 0) {
4751                     vm_deviceunplug($vmid, $conf, 'tablet');
4752                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4753                 }
4754             } elsif ($opt =~ m/^usb\d+$/) {
4755                 die "skip\n";
4756                 # since we cannot reliably hot unplug usb devices
4757                 # we are disabling it
4758                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4759                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4760                 die "skip\n" if !$d;
4761                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4762             } elsif ($opt eq 'vcpus') {
4763                 die "skip\n" if !$hotplug_features->{cpu};
4764                 qemu_cpu_hotplug($vmid, $conf, $value);
4765             } elsif ($opt eq 'balloon') {
4766                 # enable/disable balloning device is not hotpluggable
4767                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4768                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4769                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4770
4771                 # allow manual ballooning if shares is set to zero
4772                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4773                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4774                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4775                 }
4776             } elsif ($opt =~ m/^net(\d+)$/) {
4777                 # some changes can be done without hotplug
4778                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4779                                     $vmid, $opt, $value, $arch, $machine_type);
4780             } elsif (is_valid_drivename($opt)) {
4781                 # some changes can be done without hotplug
4782                 my $drive = parse_drive($opt, $value);
4783                 if (drive_is_cloudinit($drive)) {
4784                     &$apply_pending_cloudinit($opt, $value);
4785                 }
4786                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4787                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4788             } elsif ($opt =~ m/^memory$/) { #dimms
4789                 die "skip\n" if !$hotplug_features->{memory};
4790                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4791             } elsif ($opt eq 'cpuunits') {
4792                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4793             } elsif ($opt eq 'cpulimit') {
4794                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4795                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4796             } else {
4797                 die "skip\n";  # skip non-hot-pluggable options
4798             }
4799         };
4800         if (my $err = $@) {
4801             &$add_error($opt, $err) if $err ne "skip\n";
4802         } else {
4803             # save new config if hotplug was successful
4804             $conf->{$opt} = $value;
4805             delete $conf->{pending}->{$opt};
4806             PVE::QemuConfig->write_config($vmid, $conf);
4807             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4808         }
4809     }
4810 }
4811
4812 sub try_deallocate_drive {
4813     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4814
4815     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4816         my $volid = $drive->{file};
4817         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4818             my $sid = PVE::Storage::parse_volume_id($volid);
4819             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4820
4821             # check if the disk is really unused
4822             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4823                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4824             PVE::Storage::vdisk_free($storecfg, $volid);
4825             return 1;
4826         } else {
4827             # If vm is not owner of this disk remove from config
4828             return 1;
4829         }
4830     }
4831
4832     return undef;
4833 }
4834
4835 sub vmconfig_delete_or_detach_drive {
4836     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4837
4838     my $drive = parse_drive($opt, $conf->{$opt});
4839
4840     my $rpcenv = PVE::RPCEnvironment::get();
4841     my $authuser = $rpcenv->get_user();
4842
4843     if ($force) {
4844         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4845         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4846     } else {
4847         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4848     }
4849 }
4850
4851 sub vmconfig_apply_pending {
4852     my ($vmid, $conf, $storecfg) = @_;
4853
4854     # cold plug
4855
4856     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4857     while (my ($opt, $force) = each %$pending_delete_hash) {
4858         die "internal error" if $opt =~ m/^unused/;
4859         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4860         if (!defined($conf->{$opt})) {
4861             vmconfig_undelete_pending_option($conf, $opt);
4862             PVE::QemuConfig->write_config($vmid, $conf);
4863         } elsif (is_valid_drivename($opt)) {
4864             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4865             vmconfig_undelete_pending_option($conf, $opt);
4866             delete $conf->{$opt};
4867             PVE::QemuConfig->write_config($vmid, $conf);
4868         } else {
4869             vmconfig_undelete_pending_option($conf, $opt);
4870             delete $conf->{$opt};
4871             PVE::QemuConfig->write_config($vmid, $conf);
4872         }
4873     }
4874
4875     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4876
4877     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4878         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4879
4880         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4881             # skip if nothing changed
4882         } elsif (is_valid_drivename($opt)) {
4883             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4884                 if defined($conf->{$opt});
4885             $conf->{$opt} = $conf->{pending}->{$opt};
4886         } else {
4887             $conf->{$opt} = $conf->{pending}->{$opt};
4888         }
4889
4890         delete $conf->{pending}->{$opt};
4891         PVE::QemuConfig->write_config($vmid, $conf);
4892     }
4893 }
4894
4895 my $safe_num_ne = sub {
4896     my ($a, $b) = @_;
4897
4898     return 0 if !defined($a) && !defined($b);
4899     return 1 if !defined($a);
4900     return 1 if !defined($b);
4901
4902     return $a != $b;
4903 };
4904
4905 my $safe_string_ne = sub {
4906     my ($a, $b) = @_;
4907
4908     return 0 if !defined($a) && !defined($b);
4909     return 1 if !defined($a);
4910     return 1 if !defined($b);
4911
4912     return $a ne $b;
4913 };
4914
4915 sub vmconfig_update_net {
4916     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4917
4918     my $newnet = parse_net($value);
4919
4920     if ($conf->{$opt}) {
4921         my $oldnet = parse_net($conf->{$opt});
4922
4923         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4924             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4925             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4926             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4927
4928             # for non online change, we try to hot-unplug
4929             die "skip\n" if !$hotplug;
4930             vm_deviceunplug($vmid, $conf, $opt);
4931         } else {
4932
4933             die "internal error" if $opt !~ m/net(\d+)/;
4934             my $iface = "tap${vmid}i$1";
4935
4936             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4937                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4938                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4939                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4940                 PVE::Network::tap_unplug($iface);
4941                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4942             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4943                 # Rate can be applied on its own but any change above needs to
4944                 # include the rate in tap_plug since OVS resets everything.
4945                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4946             }
4947
4948             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4949                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4950             }
4951
4952             return 1;
4953         }
4954     }
4955
4956     if ($hotplug) {
4957         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4958     } else {
4959         die "skip\n";
4960     }
4961 }
4962
4963 sub vmconfig_update_disk {
4964     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4965
4966     # fixme: do we need force?
4967
4968     my $drive = parse_drive($opt, $value);
4969
4970     if ($conf->{$opt}) {
4971
4972         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4973
4974             my $media = $drive->{media} || 'disk';
4975             my $oldmedia = $old_drive->{media} || 'disk';
4976             die "unable to change media type\n" if $media ne $oldmedia;
4977
4978             if (!drive_is_cdrom($old_drive)) {
4979
4980                 if ($drive->{file} ne $old_drive->{file}) {
4981
4982                     die "skip\n" if !$hotplug;
4983
4984                     # unplug and register as unused
4985                     vm_deviceunplug($vmid, $conf, $opt);
4986                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4987
4988                 } else {
4989                     # update existing disk
4990
4991                     # skip non hotpluggable value
4992                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4993                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4994                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4995                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4996                         die "skip\n";
4997                     }
4998
4999                     # apply throttle
5000                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5001                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5002                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5003                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5004                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5005                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5006                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5007                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5008                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5009                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5010                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5011                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5012                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5013                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5014                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5015                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5016                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5017                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5018
5019                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5020                                                    ($drive->{mbps} || 0)*1024*1024,
5021                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5022                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5023                                                    $drive->{iops} || 0,
5024                                                    $drive->{iops_rd} || 0,
5025                                                    $drive->{iops_wr} || 0,
5026                                                    ($drive->{mbps_max} || 0)*1024*1024,
5027                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5028                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5029                                                    $drive->{iops_max} || 0,
5030                                                    $drive->{iops_rd_max} || 0,
5031                                                    $drive->{iops_wr_max} || 0,
5032                                                    $drive->{bps_max_length} || 1,
5033                                                    $drive->{bps_rd_max_length} || 1,
5034                                                    $drive->{bps_wr_max_length} || 1,
5035                                                    $drive->{iops_max_length} || 1,
5036                                                    $drive->{iops_rd_max_length} || 1,
5037                                                    $drive->{iops_wr_max_length} || 1);
5038
5039                     }
5040
5041                     return 1;
5042                 }
5043
5044             } else { # cdrom
5045
5046                 if ($drive->{file} eq 'none') {
5047                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5048                     if (drive_is_cloudinit($old_drive)) {
5049                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5050                     }
5051                 } else {
5052                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5053                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5054                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5055                 }
5056
5057                 return 1;
5058             }
5059         }
5060     }
5061
5062     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5063     # hotplug new disks
5064     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5065     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5066 }
5067
5068 sub vm_start {
5069     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5070         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5071
5072     PVE::QemuConfig->lock_config($vmid, sub {
5073         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5074
5075         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5076
5077         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5078
5079         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5080
5081         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5082             vmconfig_apply_pending($vmid, $conf, $storecfg);
5083             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5084         }
5085
5086         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5087
5088         my $defaults = load_defaults();
5089
5090         # set environment variable useful inside network script
5091         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5092
5093         my $local_volumes = {};
5094
5095         if ($targetstorage) {
5096             foreach_drive($conf, sub {
5097                 my ($ds, $drive) = @_;
5098
5099                 return if drive_is_cdrom($drive);
5100
5101                 my $volid = $drive->{file};
5102
5103                 return if !$volid;
5104
5105                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5106
5107                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5108                 return if $scfg->{shared};
5109                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5110             });
5111
5112             my $format = undef;
5113
5114             foreach my $opt (sort keys %$local_volumes) {
5115
5116                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5117                 my $drive = parse_drive($opt, $conf->{$opt});
5118
5119                 #if remote storage is specified, use default format
5120                 if ($targetstorage && $targetstorage ne "1") {
5121                     $storeid = $targetstorage;
5122                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5123                     $format = $defFormat;
5124                 } else {
5125                     #else we use same format than original
5126                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5127                     $format = qemu_img_format($scfg, $volid);
5128                 }
5129
5130                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5131                 my $newdrive = $drive;
5132                 $newdrive->{format} = $format;
5133                 $newdrive->{file} = $newvolid;
5134                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5135                 $local_volumes->{$opt} = $drivestr;
5136                 #pass drive to conf for command line
5137                 $conf->{$opt} = $drivestr;
5138             }
5139         }
5140
5141         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5142
5143         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5144
5145         my $migrate_port = 0;
5146         my $migrate_uri;
5147         if ($statefile) {
5148             if ($statefile eq 'tcp') {
5149                 my $localip = "localhost";
5150                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5151                 my $nodename = PVE::INotify::nodename();
5152
5153                 if (!defined($migration_type)) {
5154                     if (defined($datacenterconf->{migration}->{type})) {
5155                         $migration_type = $datacenterconf->{migration}->{type};
5156                     } else {
5157                         $migration_type = 'secure';
5158                     }
5159                 }
5160
5161                 if ($migration_type eq 'insecure') {
5162                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5163                     if ($migrate_network_addr) {
5164                         $localip = $migrate_network_addr;
5165                     } else {
5166                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5167                     }
5168
5169                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5170                 }
5171
5172                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5173                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5174                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5175                 push @$cmd, '-incoming', $migrate_uri;
5176                 push @$cmd, '-S';
5177
5178             } elsif ($statefile eq 'unix') {
5179                 # should be default for secure migrations as a ssh TCP forward
5180                 # tunnel is not deterministic reliable ready and fails regurarly
5181                 # to set up in time, so use UNIX socket forwards
5182                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5183                 unlink $socket_addr;
5184
5185                 $migrate_uri = "unix:$socket_addr";
5186
5187                 push @$cmd, '-incoming', $migrate_uri;
5188                 push @$cmd, '-S';
5189
5190             } else {
5191                 push @$cmd, '-loadstate', $statefile;
5192             }
5193         } elsif ($paused) {
5194             push @$cmd, '-S';
5195         }
5196
5197         # host pci devices
5198         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5199           my $d = parse_hostpci($conf->{"hostpci$i"});
5200           next if !$d;
5201           my $pcidevices = $d->{pciid};
5202           foreach my $pcidevice (@$pcidevices) {
5203                 my $pciid = $pcidevice->{id};
5204
5205                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5206                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5207                 die "no pci device info for device '$pciid'\n" if !$info;
5208
5209                 if ($d->{mdev}) {
5210                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5211                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5212                 } else {
5213                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5214                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5215                     die "can't reset pci device '$pciid'\n"
5216                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5217                 }
5218           }
5219         }
5220
5221         PVE::Storage::activate_volumes($storecfg, $vollist);
5222
5223         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5224             eval {
5225                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5226                     outfunc => sub {}, errfunc => sub {});
5227             };
5228         }
5229
5230         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5231                                                   : $defaults->{cpuunits};
5232
5233         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5234         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5235
5236         my %properties = (
5237             Slice => 'qemu.slice',
5238             KillMode => 'none',
5239             CPUShares => $cpuunits
5240         );
5241
5242         if (my $cpulimit = $conf->{cpulimit}) {
5243             $properties{CPUQuota} = int($cpulimit * 100);
5244         }
5245         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5246
5247         my $run_qemu = sub {
5248             PVE::Tools::run_fork sub {
5249                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5250                 run_command($cmd, %run_params);
5251             };
5252         };
5253
5254         if ($conf->{hugepages}) {
5255
5256             my $code = sub {
5257                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5258                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5259
5260                 PVE::QemuServer::Memory::hugepages_mount();
5261                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5262
5263                 eval { $run_qemu->() };
5264                 if (my $err = $@) {
5265                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5266                     die $err;
5267                 }
5268
5269                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5270             };
5271             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5272
5273         } else {
5274             eval { $run_qemu->() };
5275         }
5276
5277         if (my $err = $@) {
5278             # deactivate volumes if start fails
5279             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5280             die "start failed: $err";
5281         }
5282
5283         print "migration listens on $migrate_uri\n" if $migrate_uri;
5284
5285         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5286             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5287             warn $@ if $@;
5288         }
5289
5290         #start nbd server for storage migration
5291         if ($targetstorage) {
5292             my $nodename = PVE::INotify::nodename();
5293             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5294             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5295             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5296             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5297
5298             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5299
5300             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5301
5302             foreach my $opt (sort keys %$local_volumes) {
5303                 my $volid = $local_volumes->{$opt};
5304                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5305                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5306                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5307             }
5308         }
5309
5310         if ($migratedfrom) {
5311             eval {
5312                 set_migration_caps($vmid);
5313             };
5314             warn $@ if $@;
5315
5316             if ($spice_port) {
5317                 print "spice listens on port $spice_port\n";
5318                 if ($spice_ticket) {
5319                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5320                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5321                 }
5322             }
5323
5324         } else {
5325             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5326                 if !$statefile && $conf->{balloon};
5327
5328             foreach my $opt (keys %$conf) {
5329                 next if $opt !~  m/^net\d+$/;
5330                 my $nicconf = parse_net($conf->{$opt});
5331                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5332             }
5333         }
5334
5335         vm_mon_cmd_nocheck($vmid, 'qom-set',
5336                     path => "machine/peripheral/balloon0",
5337                     property => "guest-stats-polling-interval",
5338                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5339
5340         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5341     });
5342 }
5343
5344 sub vm_mon_cmd {
5345     my ($vmid, $execute, %params) = @_;
5346
5347     my $cmd = { execute => $execute, arguments => \%params };
5348     vm_qmp_command($vmid, $cmd);
5349 }
5350
5351 sub vm_mon_cmd_nocheck {
5352     my ($vmid, $execute, %params) = @_;
5353
5354     my $cmd = { execute => $execute, arguments => \%params };
5355     vm_qmp_command($vmid, $cmd, 1);
5356 }
5357
5358 sub vm_qmp_command {
5359     my ($vmid, $cmd, $nocheck) = @_;
5360
5361     my $res;
5362
5363     my $timeout;
5364     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5365         $timeout = $cmd->{arguments}->{timeout};
5366         delete $cmd->{arguments}->{timeout};
5367     }
5368
5369     eval {
5370         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5371         my $sname = qmp_socket($vmid);
5372         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5373             my $qmpclient = PVE::QMPClient->new();
5374
5375             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5376         } else {
5377             die "unable to open monitor socket\n";
5378         }
5379     };
5380     if (my $err = $@) {
5381         syslog("err", "VM $vmid qmp command failed - $err");
5382         die $err;
5383     }
5384
5385     return $res;
5386 }
5387
5388 sub vm_human_monitor_command {
5389     my ($vmid, $cmdline) = @_;
5390
5391     my $res;
5392
5393     my $cmd = {
5394         execute => 'human-monitor-command',
5395         arguments => { 'command-line' => $cmdline},
5396     };
5397
5398     return vm_qmp_command($vmid, $cmd);
5399 }
5400
5401 sub vm_commandline {
5402     my ($storecfg, $vmid, $snapname) = @_;
5403
5404     my $conf = PVE::QemuConfig->load_config($vmid);
5405
5406     if ($snapname) {
5407         my $snapshot = $conf->{snapshots}->{$snapname};
5408         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5409
5410         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5411
5412         $conf = $snapshot;
5413     }
5414
5415     my $defaults = load_defaults();
5416
5417     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5418
5419     return PVE::Tools::cmd2string($cmd);
5420 }
5421
5422 sub vm_reset {
5423     my ($vmid, $skiplock) = @_;
5424
5425     PVE::QemuConfig->lock_config($vmid, sub {
5426
5427         my $conf = PVE::QemuConfig->load_config($vmid);
5428
5429         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5430
5431         vm_mon_cmd($vmid, "system_reset");
5432     });
5433 }
5434
5435 sub get_vm_volumes {
5436     my ($conf) = @_;
5437
5438     my $vollist = [];
5439     foreach_volid($conf, sub {
5440         my ($volid, $attr) = @_;
5441
5442         return if $volid =~ m|^/|;
5443
5444         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5445         return if !$sid;
5446
5447         push @$vollist, $volid;
5448     });
5449
5450     return $vollist;
5451 }
5452
5453 sub vm_stop_cleanup {
5454     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5455
5456     eval {
5457
5458         if (!$keepActive) {
5459             my $vollist = get_vm_volumes($conf);
5460             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5461         }
5462
5463         foreach my $ext (qw(mon qmp pid vnc qga)) {
5464             unlink "/var/run/qemu-server/${vmid}.$ext";
5465         }
5466
5467         foreach my $key (keys %$conf) {
5468             next if $key !~ m/^hostpci(\d+)$/;
5469             my $hostpciindex = $1;
5470             my $d = parse_hostpci($conf->{$key});
5471             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5472
5473             foreach my $pci (@{$d->{pciid}}) {
5474                 my $pciid = $pci->{id};
5475                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5476             }
5477         }
5478
5479         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5480     };
5481     warn $@ if $@; # avoid errors - just warn
5482 }
5483
5484 # Note: use $nockeck to skip tests if VM configuration file exists.
5485 # We need that when migration VMs to other nodes (files already moved)
5486 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5487 sub vm_stop {
5488     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5489
5490     $force = 1 if !defined($force) && !$shutdown;
5491
5492     if ($migratedfrom){
5493         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5494         kill 15, $pid if $pid;
5495         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5496         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5497         return;
5498     }
5499
5500     PVE::QemuConfig->lock_config($vmid, sub {
5501
5502         my $pid = check_running($vmid, $nocheck);
5503         return if !$pid;
5504
5505         my $conf;
5506         if (!$nocheck) {
5507             $conf = PVE::QemuConfig->load_config($vmid);
5508             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5509             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5510                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5511                 $timeout = $opts->{down} if $opts->{down};
5512             }
5513             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5514         }
5515
5516         $timeout = 60 if !defined($timeout);
5517
5518         eval {
5519             if ($shutdown) {
5520                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5521                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5522                 } else {
5523                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5524                 }
5525             } else {
5526                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5527             }
5528         };
5529         my $err = $@;
5530
5531         if (!$err) {
5532             my $count = 0;
5533             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5534                 $count++;
5535                 sleep 1;
5536             }
5537
5538             if ($count >= $timeout) {
5539                 if ($force) {
5540                     warn "VM still running - terminating now with SIGTERM\n";
5541                     kill 15, $pid;
5542                 } else {
5543                     die "VM quit/powerdown failed - got timeout\n";
5544                 }
5545             } else {
5546                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5547                 return;
5548             }
5549         } else {
5550             if ($force) {
5551                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5552                 kill 15, $pid;
5553             } else {
5554                 die "VM quit/powerdown failed\n";
5555             }
5556         }
5557
5558         # wait again
5559         $timeout = 10;
5560
5561         my $count = 0;
5562         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5563             $count++;
5564             sleep 1;
5565         }
5566
5567         if ($count >= $timeout) {
5568             warn "VM still running - terminating now with SIGKILL\n";
5569             kill 9, $pid;
5570             sleep 1;
5571         }
5572
5573         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5574    });
5575 }
5576
5577 sub vm_suspend {
5578     my ($vmid, $skiplock) = @_;
5579
5580     PVE::QemuConfig->lock_config($vmid, sub {
5581
5582         my $conf = PVE::QemuConfig->load_config($vmid);
5583
5584         PVE::QemuConfig->check_lock($conf)
5585             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5586
5587         vm_mon_cmd($vmid, "stop");
5588     });
5589 }
5590
5591 sub vm_resume {
5592     my ($vmid, $skiplock, $nocheck) = @_;
5593
5594     PVE::QemuConfig->lock_config($vmid, sub {
5595
5596         my $res = vm_mon_cmd($vmid, 'query-status');
5597         my $resume_cmd = 'cont';
5598
5599         if ($res->{status} && $res->{status} eq 'suspended') {
5600             $resume_cmd = 'system_wakeup';
5601         }
5602
5603         if (!$nocheck) {
5604
5605             my $conf = PVE::QemuConfig->load_config($vmid);
5606
5607             PVE::QemuConfig->check_lock($conf)
5608                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5609
5610             vm_mon_cmd($vmid, $resume_cmd);
5611
5612         } else {
5613             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5614         }
5615     });
5616 }
5617
5618 sub vm_sendkey {
5619     my ($vmid, $skiplock, $key) = @_;
5620
5621     PVE::QemuConfig->lock_config($vmid, sub {
5622
5623         my $conf = PVE::QemuConfig->load_config($vmid);
5624
5625         # there is no qmp command, so we use the human monitor command
5626         vm_human_monitor_command($vmid, "sendkey $key");
5627     });
5628 }
5629
5630 sub vm_destroy {
5631     my ($storecfg, $vmid, $skiplock) = @_;
5632
5633     PVE::QemuConfig->lock_config($vmid, sub {
5634
5635         my $conf = PVE::QemuConfig->load_config($vmid);
5636
5637         if (!check_running($vmid)) {
5638             destroy_vm($storecfg, $vmid, undef, $skiplock);
5639         } else {
5640             die "VM $vmid is running - destroy failed\n";
5641         }
5642     });
5643 }
5644
5645 # vzdump restore implementaion
5646
5647 sub tar_archive_read_firstfile {
5648     my $archive = shift;
5649
5650     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5651
5652     # try to detect archive type first
5653     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5654         die "unable to open file '$archive'\n";
5655     my $firstfile = <$fh>;
5656     kill 15, $pid;
5657     close $fh;
5658
5659     die "ERROR: archive contaions no data\n" if !$firstfile;
5660     chomp $firstfile;
5661
5662     return $firstfile;
5663 }
5664
5665 sub tar_restore_cleanup {
5666     my ($storecfg, $statfile) = @_;
5667
5668     print STDERR "starting cleanup\n";
5669
5670     if (my $fd = IO::File->new($statfile, "r")) {
5671         while (defined(my $line = <$fd>)) {
5672             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5673                 my $volid = $2;
5674                 eval {
5675                     if ($volid =~ m|^/|) {
5676                         unlink $volid || die 'unlink failed\n';
5677                     } else {
5678                         PVE::Storage::vdisk_free($storecfg, $volid);
5679                     }
5680                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5681                 };
5682                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5683             } else {
5684                 print STDERR "unable to parse line in statfile - $line";
5685             }
5686         }
5687         $fd->close();
5688     }
5689 }
5690
5691 sub restore_archive {
5692     my ($archive, $vmid, $user, $opts) = @_;
5693
5694     my $format = $opts->{format};
5695     my $comp;
5696
5697     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5698         $format = 'tar' if !$format;
5699         $comp = 'gzip';
5700     } elsif ($archive =~ m/\.tar$/) {
5701         $format = 'tar' if !$format;
5702     } elsif ($archive =~ m/.tar.lzo$/) {
5703         $format = 'tar' if !$format;
5704         $comp = 'lzop';
5705     } elsif ($archive =~ m/\.vma$/) {
5706         $format = 'vma' if !$format;
5707     } elsif ($archive =~ m/\.vma\.gz$/) {
5708         $format = 'vma' if !$format;
5709         $comp = 'gzip';
5710     } elsif ($archive =~ m/\.vma\.lzo$/) {
5711         $format = 'vma' if !$format;
5712         $comp = 'lzop';
5713     } else {
5714         $format = 'vma' if !$format; # default
5715     }
5716
5717     # try to detect archive format
5718     if ($format eq 'tar') {
5719         return restore_tar_archive($archive, $vmid, $user, $opts);
5720     } else {
5721         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5722     }
5723 }
5724
5725 sub restore_update_config_line {
5726     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5727
5728     return if $line =~ m/^\#qmdump\#/;
5729     return if $line =~ m/^\#vzdump\#/;
5730     return if $line =~ m/^lock:/;
5731     return if $line =~ m/^unused\d+:/;
5732     return if $line =~ m/^parent:/;
5733     return if $line =~ m/^template:/; # restored VM is never a template
5734
5735     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5736     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5737         # try to convert old 1.X settings
5738         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5739         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5740             my ($model, $macaddr) = split(/\=/, $devconfig);
5741             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5742             my $net = {
5743                 model => $model,
5744                 bridge => "vmbr$ind",
5745                 macaddr => $macaddr,
5746             };
5747             my $netstr = print_net($net);
5748
5749             print $outfd "net$cookie->{netcount}: $netstr\n";
5750             $cookie->{netcount}++;
5751         }
5752     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5753         my ($id, $netstr) = ($1, $2);
5754         my $net = parse_net($netstr);
5755         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5756         $netstr = print_net($net);
5757         print $outfd "$id: $netstr\n";
5758     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5759         my $virtdev = $1;
5760         my $value = $3;
5761         my $di = parse_drive($virtdev, $value);
5762         if (defined($di->{backup}) && !$di->{backup}) {
5763             print $outfd "#$line";
5764         } elsif ($map->{$virtdev}) {
5765             delete $di->{format}; # format can change on restore
5766             $di->{file} = $map->{$virtdev};
5767             $value = print_drive($vmid, $di);
5768             print $outfd "$virtdev: $value\n";
5769         } else {
5770             print $outfd $line;
5771         }
5772     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5773         my $vmgenid = $1;
5774         if ($vmgenid ne '0') {
5775             # always generate a new vmgenid if there was a valid one setup
5776             $vmgenid = generate_uuid();
5777         }
5778         print $outfd "vmgenid: $vmgenid\n";
5779     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5780         my ($uuid, $uuid_str);
5781         UUID::generate($uuid);
5782         UUID::unparse($uuid, $uuid_str);
5783         my $smbios1 = parse_smbios1($2);
5784         $smbios1->{uuid} = $uuid_str;
5785         print $outfd $1.print_smbios1($smbios1)."\n";
5786     } else {
5787         print $outfd $line;
5788     }
5789 }
5790
5791 sub scan_volids {
5792     my ($cfg, $vmid) = @_;
5793
5794     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5795
5796     my $volid_hash = {};
5797     foreach my $storeid (keys %$info) {
5798         foreach my $item (@{$info->{$storeid}}) {
5799             next if !($item->{volid} && $item->{size});
5800             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5801             $volid_hash->{$item->{volid}} = $item;
5802         }
5803     }
5804
5805     return $volid_hash;
5806 }
5807
5808 sub is_volume_in_use {
5809     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5810
5811     my $path = PVE::Storage::path($storecfg, $volid);
5812
5813     my $scan_config = sub {
5814         my ($cref, $snapname) = @_;
5815
5816         foreach my $key (keys %$cref) {
5817             my $value = $cref->{$key};
5818             if (is_valid_drivename($key)) {
5819                 next if $skip_drive && $key eq $skip_drive;
5820                 my $drive = parse_drive($key, $value);
5821                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5822                 return 1 if $volid eq $drive->{file};
5823                 if ($drive->{file} =~ m!^/!) {
5824                     return 1 if $drive->{file} eq $path;
5825                 } else {
5826                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5827                     next if !$storeid;
5828                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5829                     next if !$scfg;
5830                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5831                 }
5832             }
5833         }
5834
5835         return 0;
5836     };
5837
5838     return 1 if &$scan_config($conf);
5839
5840     undef $skip_drive;
5841
5842     foreach my $snapname (keys %{$conf->{snapshots}}) {
5843         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5844     }
5845
5846     return 0;
5847 }
5848
5849 sub update_disksize {
5850     my ($vmid, $conf, $volid_hash) = @_;
5851
5852     my $changes;
5853     my $prefix = "VM $vmid:";
5854
5855     # used and unused disks
5856     my $referenced = {};
5857
5858     # Note: it is allowed to define multiple storages with same path (alias), so
5859     # we need to check both 'volid' and real 'path' (two different volid can point
5860     # to the same path).
5861
5862     my $referencedpath = {};
5863
5864     # update size info
5865     foreach my $opt (keys %$conf) {
5866         if (is_valid_drivename($opt)) {
5867             my $drive = parse_drive($opt, $conf->{$opt});
5868             my $volid = $drive->{file};
5869             next if !$volid;
5870
5871             $referenced->{$volid} = 1;
5872             if ($volid_hash->{$volid} &&
5873                 (my $path = $volid_hash->{$volid}->{path})) {
5874                 $referencedpath->{$path} = 1;
5875             }
5876
5877             next if drive_is_cdrom($drive);
5878             next if !$volid_hash->{$volid};
5879
5880             $drive->{size} = $volid_hash->{$volid}->{size};
5881             my $new = print_drive($vmid, $drive);
5882             if ($new ne $conf->{$opt}) {
5883                 $changes = 1;
5884                 $conf->{$opt} = $new;
5885                 print "$prefix update disk '$opt' information.\n";
5886             }
5887         }
5888     }
5889
5890     # remove 'unusedX' entry if volume is used
5891     foreach my $opt (keys %$conf) {
5892         next if $opt !~ m/^unused\d+$/;
5893         my $volid = $conf->{$opt};
5894         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5895         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5896             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5897             $changes = 1;
5898             delete $conf->{$opt};
5899         }
5900
5901         $referenced->{$volid} = 1;
5902         $referencedpath->{$path} = 1 if $path;
5903     }
5904
5905     foreach my $volid (sort keys %$volid_hash) {
5906         next if $volid =~ m/vm-$vmid-state-/;
5907         next if $referenced->{$volid};
5908         my $path = $volid_hash->{$volid}->{path};
5909         next if !$path; # just to be sure
5910         next if $referencedpath->{$path};
5911         $changes = 1;
5912         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5913         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5914         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5915     }
5916
5917     return $changes;
5918 }
5919
5920 sub rescan {
5921     my ($vmid, $nolock, $dryrun) = @_;
5922
5923     my $cfg = PVE::Storage::config();
5924
5925     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5926     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5927     foreach my $stor (keys %{$cfg->{ids}}) {
5928         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5929     }
5930
5931     print "rescan volumes...\n";
5932     my $volid_hash = scan_volids($cfg, $vmid);
5933
5934     my $updatefn =  sub {
5935         my ($vmid) = @_;
5936
5937         my $conf = PVE::QemuConfig->load_config($vmid);
5938
5939         PVE::QemuConfig->check_lock($conf);
5940
5941         my $vm_volids = {};
5942         foreach my $volid (keys %$volid_hash) {
5943             my $info = $volid_hash->{$volid};
5944             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5945         }
5946
5947         my $changes = update_disksize($vmid, $conf, $vm_volids);
5948
5949         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5950     };
5951
5952     if (defined($vmid)) {
5953         if ($nolock) {
5954             &$updatefn($vmid);
5955         } else {
5956             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5957         }
5958     } else {
5959         my $vmlist = config_list();
5960         foreach my $vmid (keys %$vmlist) {
5961             if ($nolock) {
5962                 &$updatefn($vmid);
5963             } else {
5964                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5965             }
5966         }
5967     }
5968 }
5969
5970 sub restore_vma_archive {
5971     my ($archive, $vmid, $user, $opts, $comp) = @_;
5972
5973     my $readfrom = $archive;
5974
5975     my $cfg = PVE::Storage::config();
5976     my $commands = [];
5977     my $bwlimit = $opts->{bwlimit};
5978
5979     my $dbg_cmdstring = '';
5980     my $add_pipe = sub {
5981         my ($cmd) = @_;
5982         push @$commands, $cmd;
5983         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5984         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5985         $readfrom = '-';
5986     };
5987
5988     my $input = undef;
5989     if ($archive eq '-') {
5990         $input = '<&STDIN';
5991     } else {
5992         # If we use a backup from a PVE defined storage we also consider that
5993         # storage's rate limit:
5994         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5995         if (defined($volid)) {
5996             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5997             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5998             if ($readlimit) {
5999                 print STDERR "applying read rate limit: $readlimit\n";
6000                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6001                 $add_pipe->($cstream);
6002             }
6003         }
6004     }
6005
6006     if ($comp) {
6007         my $cmd;
6008         if ($comp eq 'gzip') {
6009             $cmd = ['zcat', $readfrom];
6010         } elsif ($comp eq 'lzop') {
6011             $cmd = ['lzop', '-d', '-c', $readfrom];
6012         } else {
6013             die "unknown compression method '$comp'\n";
6014         }
6015         $add_pipe->($cmd);
6016     }
6017
6018     my $tmpdir = "/var/tmp/vzdumptmp$$";
6019     rmtree $tmpdir;
6020
6021     # disable interrupts (always do cleanups)
6022     local $SIG{INT} =
6023         local $SIG{TERM} =
6024         local $SIG{QUIT} =
6025         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6026
6027     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6028     POSIX::mkfifo($mapfifo, 0600);
6029     my $fifofh;
6030
6031     my $openfifo = sub {
6032         open($fifofh, '>', $mapfifo) || die $!;
6033     };
6034
6035     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6036
6037     my $oldtimeout;
6038     my $timeout = 5;
6039
6040     my $devinfo = {};
6041
6042     my $rpcenv = PVE::RPCEnvironment::get();
6043
6044     my $conffile = PVE::QemuConfig->config_file($vmid);
6045     my $tmpfn = "$conffile.$$.tmp";
6046
6047     # Note: $oldconf is undef if VM does not exists
6048     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6049     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6050
6051     my %storage_limits;
6052
6053     my $print_devmap = sub {
6054         my $virtdev_hash = {};
6055
6056         my $cfgfn = "$tmpdir/qemu-server.conf";
6057
6058         # we can read the config - that is already extracted
6059         my $fh = IO::File->new($cfgfn, "r") ||
6060             "unable to read qemu-server.conf - $!\n";
6061
6062         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6063         if (-f $fwcfgfn) {
6064             my $pve_firewall_dir = '/etc/pve/firewall';
6065             mkdir $pve_firewall_dir; # make sure the dir exists
6066             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6067         }
6068
6069         while (defined(my $line = <$fh>)) {
6070             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6071                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6072                 die "archive does not contain data for drive '$virtdev'\n"
6073                     if !$devinfo->{$devname};
6074                 if (defined($opts->{storage})) {
6075                     $storeid = $opts->{storage} || 'local';
6076                 } elsif (!$storeid) {
6077                     $storeid = 'local';
6078                 }
6079                 $format = 'raw' if !$format;
6080                 $devinfo->{$devname}->{devname} = $devname;
6081                 $devinfo->{$devname}->{virtdev} = $virtdev;
6082                 $devinfo->{$devname}->{format} = $format;
6083                 $devinfo->{$devname}->{storeid} = $storeid;
6084
6085                 # check permission on storage
6086                 my $pool = $opts->{pool}; # todo: do we need that?
6087                 if ($user ne 'root@pam') {
6088                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6089                 }
6090
6091                 $storage_limits{$storeid} = $bwlimit;
6092
6093                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6094             }
6095         }
6096
6097         foreach my $key (keys %storage_limits) {
6098             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6099             next if !$limit;
6100             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6101             $storage_limits{$key} = $limit * 1024;
6102         }
6103
6104         foreach my $devname (keys %$devinfo) {
6105             die "found no device mapping information for device '$devname'\n"
6106                 if !$devinfo->{$devname}->{virtdev};
6107         }
6108
6109         # create empty/temp config
6110         if ($oldconf) {
6111             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6112             foreach_drive($oldconf, sub {
6113                 my ($ds, $drive) = @_;
6114
6115                 return if drive_is_cdrom($drive);
6116
6117                 my $volid = $drive->{file};
6118
6119                 return if !$volid || $volid =~ m|^/|;
6120
6121                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6122                 return if !$path || !$owner || ($owner != $vmid);
6123
6124                 # Note: only delete disk we want to restore
6125                 # other volumes will become unused
6126                 if ($virtdev_hash->{$ds}) {
6127                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6128                     if (my $err = $@) {
6129                         warn $err;
6130                     }
6131                 }
6132             });
6133
6134             # delete vmstate files
6135             # since after the restore we have no snapshots anymore
6136             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6137                 my $snap = $oldconf->{snapshots}->{$snapname};
6138                 if ($snap->{vmstate}) {
6139                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6140                     if (my $err = $@) {
6141                         warn $err;
6142                     }
6143                 }
6144             }
6145         }
6146
6147         my $map = {};
6148         foreach my $virtdev (sort keys %$virtdev_hash) {
6149             my $d = $virtdev_hash->{$virtdev};
6150             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6151             my $storeid = $d->{storeid};
6152             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6153
6154             my $map_opts = '';
6155             if (my $limit = $storage_limits{$storeid}) {
6156                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6157             }
6158
6159             # test if requested format is supported
6160             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6161             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6162             $d->{format} = $defFormat if !$supported;
6163
6164             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6165                                                   $d->{format}, undef, $alloc_size);
6166             print STDERR "new volume ID is '$volid'\n";
6167             $d->{volid} = $volid;
6168             my $path = PVE::Storage::path($cfg, $volid);
6169
6170             PVE::Storage::activate_volumes($cfg,[$volid]);
6171
6172             my $write_zeros = 1;
6173             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6174                 $write_zeros = 0;
6175             }
6176
6177             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6178
6179             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6180             $map->{$virtdev} = $volid;
6181         }
6182
6183         $fh->seek(0, 0) || die "seek failed - $!\n";
6184
6185         my $outfd = new IO::File ($tmpfn, "w") ||
6186             die "unable to write config for VM $vmid\n";
6187
6188         my $cookie = { netcount => 0 };
6189         while (defined(my $line = <$fh>)) {
6190             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6191         }
6192
6193         $fh->close();
6194         $outfd->close();
6195     };
6196
6197     eval {
6198         # enable interrupts
6199         local $SIG{INT} =
6200             local $SIG{TERM} =
6201             local $SIG{QUIT} =
6202             local $SIG{HUP} =
6203             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6204         local $SIG{ALRM} = sub { die "got timeout\n"; };
6205
6206         $oldtimeout = alarm($timeout);
6207
6208         my $parser = sub {
6209             my $line = shift;
6210
6211             print "$line\n";
6212
6213             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6214                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6215                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6216             } elsif ($line =~ m/^CTIME: /) {
6217                 # we correctly received the vma config, so we can disable
6218                 # the timeout now for disk allocation (set to 10 minutes, so
6219                 # that we always timeout if something goes wrong)
6220                 alarm(600);
6221                 &$print_devmap();
6222                 print $fifofh "done\n";
6223                 my $tmp = $oldtimeout || 0;
6224                 $oldtimeout = undef;
6225                 alarm($tmp);
6226                 close($fifofh);
6227             }
6228         };
6229
6230         print "restore vma archive: $dbg_cmdstring\n";
6231         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6232     };
6233     my $err = $@;
6234
6235     alarm($oldtimeout) if $oldtimeout;
6236
6237     my $vollist = [];
6238     foreach my $devname (keys %$devinfo) {
6239         my $volid = $devinfo->{$devname}->{volid};
6240         push @$vollist, $volid if $volid;
6241     }
6242
6243     PVE::Storage::deactivate_volumes($cfg, $vollist);
6244
6245     unlink $mapfifo;
6246
6247     if ($err) {
6248         rmtree $tmpdir;
6249         unlink $tmpfn;
6250
6251         foreach my $devname (keys %$devinfo) {
6252             my $volid = $devinfo->{$devname}->{volid};
6253             next if !$volid;
6254             eval {
6255                 if ($volid =~ m|^/|) {
6256                     unlink $volid || die 'unlink failed\n';
6257                 } else {
6258                     PVE::Storage::vdisk_free($cfg, $volid);
6259                 }
6260                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6261             };
6262             print STDERR "unable to cleanup '$volid' - $@" if $@;
6263         }
6264         die $err;
6265     }
6266
6267     rmtree $tmpdir;
6268
6269     rename($tmpfn, $conffile) ||
6270         die "unable to commit configuration file '$conffile'\n";
6271
6272     PVE::Cluster::cfs_update(); # make sure we read new file
6273
6274     eval { rescan($vmid, 1); };
6275     warn $@ if $@;
6276 }
6277
6278 sub restore_tar_archive {
6279     my ($archive, $vmid, $user, $opts) = @_;
6280
6281     if ($archive ne '-') {
6282         my $firstfile = tar_archive_read_firstfile($archive);
6283         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6284             if $firstfile ne 'qemu-server.conf';
6285     }
6286
6287     my $storecfg = PVE::Storage::config();
6288
6289     # destroy existing data - keep empty config
6290     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6291     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6292
6293     my $tocmd = "/usr/lib/qemu-server/qmextract";
6294
6295     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6296     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6297     $tocmd .= ' --prealloc' if $opts->{prealloc};
6298     $tocmd .= ' --info' if $opts->{info};
6299
6300     # tar option "xf" does not autodetect compression when read from STDIN,
6301     # so we pipe to zcat
6302     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6303         PVE::Tools::shellquote("--to-command=$tocmd");
6304
6305     my $tmpdir = "/var/tmp/vzdumptmp$$";
6306     mkpath $tmpdir;
6307
6308     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6309     local $ENV{VZDUMP_VMID} = $vmid;
6310     local $ENV{VZDUMP_USER} = $user;
6311
6312     my $conffile = PVE::QemuConfig->config_file($vmid);
6313     my $tmpfn = "$conffile.$$.tmp";
6314
6315     # disable interrupts (always do cleanups)
6316     local $SIG{INT} =
6317         local $SIG{TERM} =
6318         local $SIG{QUIT} =
6319         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6320
6321     eval {
6322         # enable interrupts
6323         local $SIG{INT} =
6324             local $SIG{TERM} =
6325             local $SIG{QUIT} =
6326             local $SIG{HUP} =
6327             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6328
6329         if ($archive eq '-') {
6330             print "extracting archive from STDIN\n";
6331             run_command($cmd, input => "<&STDIN");
6332         } else {
6333             print "extracting archive '$archive'\n";
6334             run_command($cmd);
6335         }
6336
6337         return if $opts->{info};
6338
6339         # read new mapping
6340         my $map = {};
6341         my $statfile = "$tmpdir/qmrestore.stat";
6342         if (my $fd = IO::File->new($statfile, "r")) {
6343             while (defined (my $line = <$fd>)) {
6344                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6345                     $map->{$1} = $2 if $1;
6346                 } else {
6347                     print STDERR "unable to parse line in statfile - $line\n";
6348                 }
6349             }
6350             $fd->close();
6351         }
6352
6353         my $confsrc = "$tmpdir/qemu-server.conf";
6354
6355         my $srcfd = new IO::File($confsrc, "r") ||
6356             die "unable to open file '$confsrc'\n";
6357
6358         my $outfd = new IO::File ($tmpfn, "w") ||
6359             die "unable to write config for VM $vmid\n";
6360
6361         my $cookie = { netcount => 0 };
6362         while (defined (my $line = <$srcfd>)) {
6363             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6364         }
6365
6366         $srcfd->close();
6367         $outfd->close();
6368     };
6369     my $err = $@;
6370
6371     if ($err) {
6372
6373         unlink $tmpfn;
6374
6375         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6376
6377         die $err;
6378     }
6379
6380     rmtree $tmpdir;
6381
6382     rename $tmpfn, $conffile ||
6383         die "unable to commit configuration file '$conffile'\n";
6384
6385     PVE::Cluster::cfs_update(); # make sure we read new file
6386
6387     eval { rescan($vmid, 1); };
6388     warn $@ if $@;
6389 };
6390
6391 sub foreach_storage_used_by_vm {
6392     my ($conf, $func) = @_;
6393
6394     my $sidhash = {};
6395
6396     foreach_drive($conf, sub {
6397         my ($ds, $drive) = @_;
6398         return if drive_is_cdrom($drive);
6399
6400         my $volid = $drive->{file};
6401
6402         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6403         $sidhash->{$sid} = $sid if $sid;
6404     });
6405
6406     foreach my $sid (sort keys %$sidhash) {
6407         &$func($sid);
6408     }
6409 }
6410
6411 sub do_snapshots_with_qemu {
6412     my ($storecfg, $volid) = @_;
6413
6414     my $storage_name = PVE::Storage::parse_volume_id($volid);
6415
6416     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6417         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6418         return 1;
6419     }
6420
6421     if ($volid =~ m/\.(qcow2|qed)$/){
6422         return 1;
6423     }
6424
6425     return undef;
6426 }
6427
6428 sub qga_check_running {
6429     my ($vmid, $nowarn) = @_;
6430
6431     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6432     if ($@) {
6433         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6434         return 0;
6435     }
6436     return 1;
6437 }
6438
6439 sub template_create {
6440     my ($vmid, $conf, $disk) = @_;
6441
6442     my $storecfg = PVE::Storage::config();
6443
6444     foreach_drive($conf, sub {
6445         my ($ds, $drive) = @_;
6446
6447         return if drive_is_cdrom($drive);
6448         return if $disk && $ds ne $disk;
6449
6450         my $volid = $drive->{file};
6451         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6452
6453         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6454         $drive->{file} = $voliddst;
6455         $conf->{$ds} = print_drive($vmid, $drive);
6456         PVE::QemuConfig->write_config($vmid, $conf);
6457     });
6458 }
6459
6460 sub qemu_img_convert {
6461     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6462
6463     my $storecfg = PVE::Storage::config();
6464     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6465     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6466
6467     if ($src_storeid && $dst_storeid) {
6468
6469         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6470
6471         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6472         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6473
6474         my $src_format = qemu_img_format($src_scfg, $src_volname);
6475         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6476
6477         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6478         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6479
6480         my $cmd = [];
6481         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6482         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6483         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6484         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6485         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6486         if ($is_zero_initialized) {
6487             push @$cmd, "zeroinit:$dst_path";
6488         } else {
6489             push @$cmd, $dst_path;
6490         }
6491
6492         my $parser = sub {
6493             my $line = shift;
6494             if($line =~ m/\((\S+)\/100\%\)/){
6495                 my $percent = $1;
6496                 my $transferred = int($size * $percent / 100);
6497                 my $remaining = $size - $transferred;
6498
6499                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6500             }
6501
6502         };
6503
6504         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6505         my $err = $@;
6506         die "copy failed: $err" if $err;
6507     }
6508 }
6509
6510 sub qemu_img_format {
6511     my ($scfg, $volname) = @_;
6512
6513     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6514         return $1;
6515     } else {
6516         return "raw";
6517     }
6518 }
6519
6520 sub qemu_drive_mirror {
6521     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6522
6523     $jobs = {} if !$jobs;
6524
6525     my $qemu_target;
6526     my $format;
6527     $jobs->{"drive-$drive"} = {};
6528
6529     if ($dst_volid =~ /^nbd:/) {
6530         $qemu_target = $dst_volid;
6531         $format = "nbd";
6532     } else {
6533         my $storecfg = PVE::Storage::config();
6534         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6535
6536         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6537
6538         $format = qemu_img_format($dst_scfg, $dst_volname);
6539
6540         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6541
6542         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6543     }
6544
6545     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6546     $opts->{format} = $format if $format;
6547
6548     print "drive mirror is starting for drive-$drive\n";
6549
6550     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6551
6552     if (my $err = $@) {
6553         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6554         die "mirroring error: $err";
6555     }
6556
6557     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6558 }
6559
6560 sub qemu_drive_mirror_monitor {
6561     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6562
6563     eval {
6564         my $err_complete = 0;
6565
6566         while (1) {
6567             die "storage migration timed out\n" if $err_complete > 300;
6568
6569             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6570
6571             my $running_mirror_jobs = {};
6572             foreach my $stat (@$stats) {
6573                 next if $stat->{type} ne 'mirror';
6574                 $running_mirror_jobs->{$stat->{device}} = $stat;
6575             }
6576
6577             my $readycounter = 0;
6578
6579             foreach my $job (keys %$jobs) {
6580
6581                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6582                     print "$job : finished\n";
6583                     delete $jobs->{$job};
6584                     next;
6585                 }
6586
6587                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6588
6589                 my $busy = $running_mirror_jobs->{$job}->{busy};
6590                 my $ready = $running_mirror_jobs->{$job}->{ready};
6591                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6592                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6593                     my $remaining = $total - $transferred;
6594                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6595
6596                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6597                 }
6598
6599                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6600             }
6601
6602             last if scalar(keys %$jobs) == 0;
6603
6604             if ($readycounter == scalar(keys %$jobs)) {
6605                 print "all mirroring jobs are ready \n";
6606                 last if $skipcomplete; #do the complete later
6607
6608                 if ($vmiddst && $vmiddst != $vmid) {
6609                     my $agent_running = $qga && qga_check_running($vmid);
6610                     if ($agent_running) {
6611                         print "freeze filesystem\n";
6612                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6613                     } else {
6614                         print "suspend vm\n";
6615                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6616                     }
6617
6618                     # if we clone a disk for a new target vm, we don't switch the disk
6619                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6620
6621                     if ($agent_running) {
6622                         print "unfreeze filesystem\n";
6623                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6624                     } else {
6625                         print "resume vm\n";
6626                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6627                     }
6628
6629                     last;
6630                 } else {
6631
6632                     foreach my $job (keys %$jobs) {
6633                         # try to switch the disk if source and destination are on the same guest
6634                         print "$job: Completing block job...\n";
6635
6636                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6637                         if ($@ =~ m/cannot be completed/) {
6638                             print "$job: Block job cannot be completed, try again.\n";
6639                             $err_complete++;
6640                         }else {
6641                             print "$job: Completed successfully.\n";
6642                             $jobs->{$job}->{complete} = 1;
6643                         }
6644                     }
6645                 }
6646             }
6647             sleep 1;
6648         }
6649     };
6650     my $err = $@;
6651
6652     if ($err) {
6653         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6654         die "mirroring error: $err";
6655     }
6656
6657 }
6658
6659 sub qemu_blockjobs_cancel {
6660     my ($vmid, $jobs) = @_;
6661
6662     foreach my $job (keys %$jobs) {
6663         print "$job: Cancelling block job\n";
6664         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6665         $jobs->{$job}->{cancel} = 1;
6666     }
6667
6668     while (1) {
6669         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6670
6671         my $running_jobs = {};
6672         foreach my $stat (@$stats) {
6673             $running_jobs->{$stat->{device}} = $stat;
6674         }
6675
6676         foreach my $job (keys %$jobs) {
6677
6678             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6679                 print "$job: Done.\n";
6680                 delete $jobs->{$job};
6681             }
6682         }
6683
6684         last if scalar(keys %$jobs) == 0;
6685
6686         sleep 1;
6687     }
6688 }
6689
6690 sub clone_disk {
6691     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6692         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6693
6694     my $newvolid;
6695
6696     if (!$full) {
6697         print "create linked clone of drive $drivename ($drive->{file})\n";
6698         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6699         push @$newvollist, $newvolid;
6700     } else {
6701
6702         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6703         $storeid = $storage if $storage;
6704
6705         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6706         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6707
6708         print "create full clone of drive $drivename ($drive->{file})\n";
6709         my $name = undef;
6710         if (drive_is_cloudinit($drive)) {
6711             $name = "vm-$newvmid-cloudinit";
6712             $snapname = undef;
6713             # cloudinit only supports raw and qcow2 atm:
6714             if ($dst_format eq 'qcow2') {
6715                 $name .= '.qcow2';
6716             } elsif ($dst_format ne 'raw') {
6717                 die "clone: unhandled format for cloudinit image\n";
6718             }
6719         }
6720         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6721         push @$newvollist, $newvolid;
6722
6723         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6724
6725         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6726         if (!$running || $snapname) {
6727             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6728         } else {
6729
6730             my $kvmver = get_running_qemu_version ($vmid);
6731             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6732                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6733                     if $drive->{iothread};
6734             }
6735
6736             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6737         }
6738     }
6739
6740     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6741
6742     my $disk = $drive;
6743     $disk->{format} = undef;
6744     $disk->{file} = $newvolid;
6745     $disk->{size} = $size;
6746
6747     return $disk;
6748 }
6749
6750 # this only works if VM is running
6751 sub get_current_qemu_machine {
6752     my ($vmid) = @_;
6753
6754     my $cmd = { execute => 'query-machines', arguments => {} };
6755     my $res = vm_qmp_command($vmid, $cmd);
6756
6757     my ($current, $default);
6758     foreach my $e (@$res) {
6759         $default = $e->{name} if $e->{'is-default'};
6760         $current = $e->{name} if $e->{'is-current'};
6761     }
6762
6763     # fallback to the default machine if current is not supported by qemu
6764     return $current || $default || 'pc';
6765 }
6766
6767 sub get_running_qemu_version {
6768     my ($vmid) = @_;
6769     my $cmd = { execute => 'query-version', arguments => {} };
6770     my $res = vm_qmp_command($vmid, $cmd);
6771     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6772 }
6773
6774 sub qemu_machine_feature_enabled {
6775     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6776
6777     my $current_major;
6778     my $current_minor;
6779
6780     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6781
6782         $current_major = $3;
6783         $current_minor = $4;
6784
6785     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6786
6787         $current_major = $1;
6788         $current_minor = $2;
6789     }
6790
6791     return 1 if $current_major > $version_major ||
6792                 ($current_major == $version_major &&
6793                  $current_minor >= $version_minor);
6794 }
6795
6796 sub qemu_machine_pxe {
6797     my ($vmid, $conf, $machine) = @_;
6798
6799     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6800
6801     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6802         $machine .= '.pxe';
6803     }
6804
6805     return $machine;
6806 }
6807
6808 sub qemu_use_old_bios_files {
6809     my ($machine_type) = @_;
6810
6811     return if !$machine_type;
6812
6813     my $use_old_bios_files = undef;
6814
6815     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6816         $machine_type = $1;
6817         $use_old_bios_files = 1;
6818     } else {
6819         my $kvmver = kvm_user_version();
6820         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6821         # load new efi bios files on migration. So this hack is required to allow
6822         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6823         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6824         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6825     }
6826
6827     return ($use_old_bios_files, $machine_type);
6828 }
6829
6830 sub create_efidisk($$$$$) {
6831     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6832
6833     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6834     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6835
6836     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6837     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6838     PVE::Storage::activate_volumes($storecfg, [$volid]);
6839
6840     my $path = PVE::Storage::path($storecfg, $volid);
6841     eval {
6842         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6843     };
6844     die "Copying EFI vars image failed: $@" if $@;
6845
6846     return ($volid, $vars_size);
6847 }
6848
6849 sub vm_iothreads_list {
6850     my ($vmid) = @_;
6851
6852     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6853
6854     my $iothreads = {};
6855     foreach my $iothread (@$res) {
6856         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6857     }
6858
6859     return $iothreads;
6860 }
6861
6862 sub scsihw_infos {
6863     my ($conf, $drive) = @_;
6864
6865     my $maxdev = 0;
6866
6867     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6868         $maxdev = 7;
6869     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6870         $maxdev = 1;
6871     } else {
6872         $maxdev = 256;
6873     }
6874
6875     my $controller = int($drive->{index} / $maxdev);
6876     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6877
6878     return ($maxdev, $controller, $controller_prefix);
6879 }
6880
6881 sub add_hyperv_enlightenments {
6882     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
6883
6884     return if $winversion < 6;
6885     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6886
6887     if ($gpu_passthrough || defined($hv_vendor_id)) {
6888         $hv_vendor_id //= 'proxmox';
6889         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
6890     }
6891
6892     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6893         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6894         push @$cpuFlags , 'hv_vapic';
6895         push @$cpuFlags , 'hv_time';
6896     } else {
6897         push @$cpuFlags , 'hv_spinlocks=0xffff';
6898     }
6899
6900     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6901         push @$cpuFlags , 'hv_reset';
6902         push @$cpuFlags , 'hv_vpindex';
6903         push @$cpuFlags , 'hv_runtime';
6904     }
6905
6906     if ($winversion >= 7) {
6907         push @$cpuFlags , 'hv_relaxed';
6908
6909         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6910             push @$cpuFlags , 'hv_synic';
6911             push @$cpuFlags , 'hv_stimer';
6912         }
6913     }
6914 }
6915
6916 sub windows_version {
6917     my ($ostype) = @_;
6918
6919     return 0 if !$ostype;
6920
6921     my $winversion = 0;
6922
6923     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6924         $winversion = 5;
6925     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6926         $winversion = 6;
6927     } elsif ($ostype =~ m/^win(\d+)$/) {
6928         $winversion = $1;
6929     }
6930
6931     return $winversion;
6932 }
6933
6934 sub resolve_dst_disk_format {
6935         my ($storecfg, $storeid, $src_volname, $format) = @_;
6936         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6937
6938         if (!$format) {
6939             # if no target format is specified, use the source disk format as hint
6940             if ($src_volname) {
6941                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6942                 $format = qemu_img_format($scfg, $src_volname);
6943             } else {
6944                 return $defFormat;
6945             }
6946         }
6947
6948         # test if requested format is supported - else use default
6949         my $supported = grep { $_ eq $format } @$validFormats;
6950         $format = $defFormat if !$supported;
6951         return $format;
6952 }
6953
6954 sub resolve_first_disk {
6955     my $conf = shift;
6956     my @disks = PVE::QemuServer::valid_drive_names();
6957     my $firstdisk;
6958     foreach my $ds (reverse @disks) {
6959         next if !$conf->{$ds};
6960         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6961         next if PVE::QemuServer::drive_is_cdrom($disk);
6962         $firstdisk = $ds;
6963     }
6964     return $firstdisk;
6965 }
6966
6967 sub generate_uuid {
6968     my ($uuid, $uuid_str);
6969     UUID::generate($uuid);
6970     UUID::unparse($uuid, $uuid_str);
6971     return $uuid_str;
6972 }
6973
6974 sub generate_smbios1_uuid {
6975     return "uuid=".generate_uuid();
6976 }
6977
6978 sub nbd_stop {
6979     my ($vmid) = @_;
6980
6981     vm_mon_cmd($vmid, 'nbd-server-stop');
6982 }
6983
6984 # bash completion helper
6985
6986 sub complete_backup_archives {
6987     my ($cmdname, $pname, $cvalue) = @_;
6988
6989     my $cfg = PVE::Storage::config();
6990
6991     my $storeid;
6992
6993     if ($cvalue =~ m/^([^:]+):/) {
6994         $storeid = $1;
6995     }
6996
6997     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6998
6999     my $res = [];
7000     foreach my $id (keys %$data) {
7001         foreach my $item (@{$data->{$id}}) {
7002             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7003             push @$res, $item->{volid} if defined($item->{volid});
7004         }
7005     }
7006
7007     return $res;
7008 }
7009
7010 my $complete_vmid_full = sub {
7011     my ($running) = @_;
7012
7013     my $idlist = vmstatus();
7014
7015     my $res = [];
7016
7017     foreach my $id (keys %$idlist) {
7018         my $d = $idlist->{$id};
7019         if (defined($running)) {
7020             next if $d->{template};
7021             next if $running && $d->{status} ne 'running';
7022             next if !$running && $d->{status} eq 'running';
7023         }
7024         push @$res, $id;
7025
7026     }
7027     return $res;
7028 };
7029
7030 sub complete_vmid {
7031     return &$complete_vmid_full();
7032 }
7033
7034 sub complete_vmid_stopped {
7035     return &$complete_vmid_full(0);
7036 }
7037
7038 sub complete_vmid_running {
7039     return &$complete_vmid_full(1);
7040 }
7041
7042 sub complete_storage {
7043
7044     my $cfg = PVE::Storage::config();
7045     my $ids = $cfg->{ids};
7046
7047     my $res = [];
7048     foreach my $sid (keys %$ids) {
7049         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7050         next if !$ids->{$sid}->{content}->{images};
7051         push @$res, $sid;
7052     }
7053
7054     return $res;
7055 }
7056
7057 1;