PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use Time::HiRes qw(gettimeofday);
  40 use File::Copy qw(copy);
  41 use URI::Escape;
  42
  43 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  44 my $OVMF = {
  45     x86_64 => [
  46         "$EDK2_FW_BASE/OVMF_CODE.fd",
  47         "$EDK2_FW_BASE/OVMF_VARS.fd"
  48     ],
  49     aarch64 => [
  50         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  51         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  52     ],
  53 };
  54
  55 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  56
  57 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  58
  59 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  60
  61 # Note about locking: we use flock on the config file protect
  62 # against concurent actions.
  63 # Aditionaly, we have a 'lock' setting in the config file. This
  64 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  65 # allowed when such lock is set. But you can ignore this kind of
  66 # lock with the --skiplock flag.
  67
  68 cfs_register_file('/qemu-server/',
  69                   \&parse_vm_config,
  70                   \&write_vm_config);
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  73     description => "Some command save/restore state from this location.",
  74     type => 'string',
  75     maxLength => 128,
  76     optional => 1,
  77 });
  78
  79 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  80     description => "The name of the snapshot.",
  81     type => 'string', format => 'pve-configid',
  82     maxLength => 40,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $cpu_vendor_list = {
 123     # Intel CPUs
 124     486 => 'GenuineIntel',
 125     pentium => 'GenuineIntel',
 126     pentium2  => 'GenuineIntel',
 127     pentium3  => 'GenuineIntel',
 128     coreduo => 'GenuineIntel',
 129     core2duo => 'GenuineIntel',
 130     Conroe  => 'GenuineIntel',
 131     Penryn  => 'GenuineIntel',
 132     Nehalem  => 'GenuineIntel',
 133     'Nehalem-IBRS'  => 'GenuineIntel',
 134     Westmere => 'GenuineIntel',
 135     'Westmere-IBRS' => 'GenuineIntel',
 136     SandyBridge => 'GenuineIntel',
 137     'SandyBridge-IBRS' => 'GenuineIntel',
 138     IvyBridge => 'GenuineIntel',
 139     'IvyBridge-IBRS' => 'GenuineIntel',
 140     Haswell => 'GenuineIntel',
 141     'Haswell-IBRS' => 'GenuineIntel',
 142     'Haswell-noTSX' => 'GenuineIntel',
 143     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 144     Broadwell => 'GenuineIntel',
 145     'Broadwell-IBRS' => 'GenuineIntel',
 146     'Broadwell-noTSX' => 'GenuineIntel',
 147     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 148     'Skylake-Client' => 'GenuineIntel',
 149     'Skylake-Client-IBRS' => 'GenuineIntel',
 150     'Skylake-Server' => 'GenuineIntel',
 151     'Skylake-Server-IBRS' => 'GenuineIntel',
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 174
 175 my $cpu_fmt = {
 176     cputype => {
 177         description => "Emulated CPU type.",
 178         type => 'string',
 179         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 180         default => 'kvm64',
 181         default_key => 1,
 182     },
 183     hidden => {
 184         description => "Do not identify as a KVM virtual machine.",
 185         type => 'boolean',
 186         optional => 1,
 187         default => 0
 188     },
 189     'hv-vendor-id' => {
 190         type => 'string',
 191         pattern => qr/[a-zA-Z0-9]{1,12}/,
 192         format_description => 'vendor-id',
 193         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 194         optional => 1,
 195     },
 196     flags => {
 197         description => "List of additional CPU flags separated by ';'."
 198                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 199                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 200         format_description => '+FLAG[;-FLAG...]',
 201         type => 'string',
 202         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 203         optional => 1,
 204     },
 205 };
 206
 207 my $watchdog_fmt = {
 208     model => {
 209         default_key => 1,
 210         type => 'string',
 211         enum => [qw(i6300esb ib700)],
 212         description => "Watchdog type to emulate.",
 213         default => 'i6300esb',
 214         optional => 1,
 215     },
 216     action => {
 217         type => 'string',
 218         enum => [qw(reset shutdown poweroff pause debug none)],
 219         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 220         optional => 1,
 221     },
 222 };
 223 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 224
 225 my $agent_fmt = {
 226     enabled => {
 227         description => "Enable/disable Qemu GuestAgent.",
 228         type => 'boolean',
 229         default => 0,
 230         default_key => 1,
 231     },
 232     fstrim_cloned_disks => {
 233         description => "Run fstrim after cloning/moving a disk.",
 234         type => 'boolean',
 235         optional => 1,
 236         default => 0
 237     },
 238 };
 239
 240 my $vga_fmt = {
 241     type => {
 242         description => "Select the VGA type.",
 243         type => 'string',
 244         default => 'std',
 245         optional => 1,
 246         default_key => 1,
 247         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 248     },
 249     memory => {
 250         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 251         type => 'integer',
 252         optional => 1,
 253         minimum => 4,
 254         maximum => 512,
 255     },
 256 };
 257
 258 my $confdesc = {
 259     onboot => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Specifies whether a VM will be started during system bootup.",
 263         default => 0,
 264     },
 265     autostart => {
 266         optional => 1,
 267         type => 'boolean',
 268         description => "Automatic restart after crash (currently ignored).",
 269         default => 0,
 270     },
 271     hotplug => {
 272         optional => 1,
 273         type => 'string', format => 'pve-hotplug-features',
 274         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 275         default => 'network,disk,usb',
 276     },
 277     reboot => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 281         default => 1,
 282     },
 283     lock => {
 284         optional => 1,
 285         type => 'string',
 286         description => "Lock/unlock the VM.",
 287         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 288     },
 289     cpulimit => {
 290         optional => 1,
 291         type => 'number',
 292         description => "Limit of CPU usage.",
 293         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 294         minimum => 0,
 295         maximum => 128,
 296         default => 0,
 297     },
 298     cpuunits => {
 299         optional => 1,
 300         type => 'integer',
 301         description => "CPU weight for a VM.",
 302         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 303         minimum => 2,
 304         maximum => 262144,
 305         default => 1024,
 306     },
 307     memory => {
 308         optional => 1,
 309         type => 'integer',
 310         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 311         minimum => 16,
 312         default => 512,
 313     },
 314     balloon => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 318         minimum => 0,
 319     },
 320     shares => {
 321         optional => 1,
 322         type => 'integer',
 323         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 324         minimum => 0,
 325         maximum => 50000,
 326         default => 1000,
 327     },
 328     keyboard => {
 329         optional => 1,
 330         type => 'string',
 331         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 332                        "It should not be necessary to set it.",
 333         enum => PVE::Tools::kvmkeymaplist(),
 334         default => undef,
 335     },
 336     name => {
 337         optional => 1,
 338         type => 'string', format => 'dns-name',
 339         description => "Set a name for the VM. Only used on the configuration web interface.",
 340     },
 341     scsihw => {
 342         optional => 1,
 343         type => 'string',
 344         description => "SCSI controller model",
 345         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 346         default => 'lsi',
 347     },
 348     description => {
 349         optional => 1,
 350         type => 'string',
 351         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 352     },
 353     ostype => {
 354         optional => 1,
 355         type => 'string',
 356         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 357         description => "Specify guest operating system.",
 358         verbose_description => <<EODESC,
 359 Specify guest operating system. This is used to enable special
 360 optimization/features for specific operating systems:
 361
 362 [horizontal]
 363 other;; unspecified OS
 364 wxp;; Microsoft Windows XP
 365 w2k;; Microsoft Windows 2000
 366 w2k3;; Microsoft Windows 2003
 367 w2k8;; Microsoft Windows 2008
 368 wvista;; Microsoft Windows Vista
 369 win7;; Microsoft Windows 7
 370 win8;; Microsoft Windows 8/2012/2012r2
 371 win10;; Microsoft Windows 10/2016
 372 l24;; Linux 2.4 Kernel
 373 l26;; Linux 2.6/3.X Kernel
 374 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 375 EODESC
 376     },
 377     boot => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 381         pattern => '[acdn]{1,4}',
 382         default => 'cdn',
 383     },
 384     bootdisk => {
 385         optional => 1,
 386         type => 'string', format => 'pve-qm-bootdisk',
 387         description => "Enable booting from specified disk.",
 388         pattern => '(ide|sata|scsi|virtio)\d+',
 389     },
 390     smp => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of CPUs. Please use option -sockets instead.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     sockets => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "The number of CPU sockets.",
 401         minimum => 1,
 402         default => 1,
 403     },
 404     cores => {
 405         optional => 1,
 406         type => 'integer',
 407         description => "The number of cores per socket.",
 408         minimum => 1,
 409         default => 1,
 410     },
 411     numa => {
 412         optional => 1,
 413         type => 'boolean',
 414         description => "Enable/disable NUMA.",
 415         default => 0,
 416     },
 417     hugepages => {
 418         optional => 1,
 419         type => 'string',
 420         description => "Enable/disable hugepages memory.",
 421         enum => [qw(any 2 1024)],
 422     },
 423     vcpus => {
 424         optional => 1,
 425         type => 'integer',
 426         description => "Number of hotplugged vcpus.",
 427         minimum => 1,
 428         default => 0,
 429     },
 430     acpi => {
 431         optional => 1,
 432         type => 'boolean',
 433         description => "Enable/disable ACPI.",
 434         default => 1,
 435     },
 436     agent => {
 437         optional => 1,
 438         description => "Enable/disable Qemu GuestAgent and its properties.",
 439         type => 'string',
 440         format => $agent_fmt,
 441     },
 442     kvm => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable KVM hardware virtualization.",
 446         default => 1,
 447     },
 448     tdf => {
 449         optional => 1,
 450         type => 'boolean',
 451         description => "Enable/disable time drift fix.",
 452         default => 0,
 453     },
 454     localtime => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 458     },
 459     freeze => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 463     },
 464     vga => {
 465         optional => 1,
 466         type => 'string', format => $vga_fmt,
 467         description => "Configure the VGA hardware.",
 468         verbose_description => "Configure the VGA Hardware. If you want to use ".
 469             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 470             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 471             "is 'std' for all OS types besides some Windows versions (XP and " .
 472             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 473             "display server. For win* OS you can select how many independent " .
 474             "displays you want, Linux guests can add displays them self.\n".
 475             "You can also run without any graphic card, using a serial device as terminal.",
 476     },
 477     watchdog => {
 478         optional => 1,
 479         type => 'string', format => 'pve-qm-watchdog',
 480         description => "Create a virtual hardware watchdog device.",
 481         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 482             " (by a guest action), the watchdog must be periodically polled " .
 483             "by an agent inside the guest or else the watchdog will reset " .
 484             "the guest (or execute the respective action specified)",
 485     },
 486     startdate => {
 487         optional => 1,
 488         type => 'string',
 489         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 490         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 491         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 492         default => 'now',
 493     },
 494     startup =>  get_standard_option('pve-startup-order'),
 495     template => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable Template.",
 499         default => 0,
 500     },
 501     args => {
 502         optional => 1,
 503         type => 'string',
 504         description => "Arbitrary arguments passed to kvm.",
 505         verbose_description => <<EODESCR,
 506 Arbitrary arguments passed to kvm, for example:
 507
 508 args: -no-reboot -no-hpet
 509
 510 NOTE: this option is for experts only.
 511 EODESCR
 512     },
 513     tablet => {
 514         optional => 1,
 515         type => 'boolean',
 516         default => 1,
 517         description => "Enable/disable the USB tablet device.",
 518         verbose_description => "Enable/disable the USB tablet device. This device is " .
 519             "usually needed to allow absolute mouse positioning with VNC. " .
 520             "Else the mouse runs out of sync with normal VNC clients. " .
 521             "If you're running lots of console-only guests on one host, " .
 522             "you may consider disabling this to save some context switches. " .
 523             "This is turned off by default if you use spice (-vga=qxl).",
 524     },
 525     migrate_speed => {
 526         optional => 1,
 527         type => 'integer',
 528         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 529         minimum => 0,
 530         default => 0,
 531     },
 532     migrate_downtime => {
 533         optional => 1,
 534         type => 'number',
 535         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 536         minimum => 0,
 537         default => 0.1,
 538     },
 539     cdrom => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-ide',
 542         typetext => '<volume>',
 543         description => "This is an alias for option -ide2",
 544     },
 545     cpu => {
 546         optional => 1,
 547         description => "Emulated CPU type.",
 548         type => 'string',
 549         format => $cpu_fmt,
 550     },
 551     parent => get_standard_option('pve-snapshot-name', {
 552         optional => 1,
 553         description => "Parent snapshot name. This is used internally, and should not be modified.",
 554     }),
 555     snaptime => {
 556         optional => 1,
 557         description => "Timestamp for snapshots.",
 558         type => 'integer',
 559         minimum => 0,
 560     },
 561     vmstate => {
 562         optional => 1,
 563         type => 'string', format => 'pve-volume-id',
 564         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 565     },
 566     vmstatestorage => get_standard_option('pve-storage-id', {
 567         description => "Default storage for VM state volumes/files.",
 568         optional => 1,
 569     }),
 570     runningmachine => get_standard_option('pve-qemu-machine', {
 571         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 572     }),
 573     machine => get_standard_option('pve-qemu-machine'),
 574     arch => {
 575         description => "Virtual processor architecture. Defaults to the host.",
 576         optional => 1,
 577         type => 'string',
 578         enum => [qw(x86_64 aarch64)],
 579     },
 580     smbios1 => {
 581         description => "Specify SMBIOS type 1 fields.",
 582         type => 'string', format => 'pve-qm-smbios1',
 583         maxLength => 256,
 584         optional => 1,
 585     },
 586     protection => {
 587         optional => 1,
 588         type => 'boolean',
 589         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 590         default => 0,
 591     },
 592     bios => {
 593         optional => 1,
 594         type => 'string',
 595         enum => [ qw(seabios ovmf) ],
 596         description => "Select BIOS implementation.",
 597         default => 'seabios',
 598     },
 599     vmgenid => {
 600         type => 'string',
 601         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 602         format_description => 'UUID',
 603         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 604         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 605             " 128-bit integer value identifier to the guest OS. This allows to".
 606             " notify the guest operating system when the virtual machine is".
 607             " executed with a different configuration (e.g. snapshot execution".
 608             " or creation from a template). The guest operating system notices".
 609             " the change, and is then able to react as appropriate by marking".
 610             " its copies of distributed databases as dirty, re-initializing its".
 611             " random number generator, etc.\n".
 612             "Note that auto-creation only works when done throug API/CLI create".
 613             " or update methods, but not when manually editing the config file.",
 614         default => "1 (autogenerated)",
 615         optional => 1,
 616     },
 617 };
 618
 619 my $confdesc_cloudinit = {
 620     citype => {
 621         optional => 1,
 622         type => 'string',
 623         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 624         enum => ['configdrive2', 'nocloud'],
 625     },
 626     ciuser => {
 627         optional => 1,
 628         type => 'string',
 629         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 630     },
 631     cipassword => {
 632         optional => 1,
 633         type => 'string',
 634         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 635     },
 636     searchdomain => {
 637         optional => 1,
 638         type => 'string',
 639         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 640     },
 641     nameserver => {
 642         optional => 1,
 643         type => 'string', format => 'address-list',
 644         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 645     },
 646     sshkeys => {
 647         optional => 1,
 648         type => 'string',
 649         format => 'urlencoded',
 650         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 651     },
 652 };
 653
 654 # what about other qemu settings ?
 655 #cpu => 'string',
 656 #machine => 'string',
 657 #fda => 'file',
 658 #fdb => 'file',
 659 #mtdblock => 'file',
 660 #sd => 'file',
 661 #pflash => 'file',
 662 #snapshot => 'bool',
 663 #bootp => 'file',
 664 ##tftp => 'dir',
 665 ##smb => 'dir',
 666 #kernel => 'file',
 667 #append => 'string',
 668 #initrd => 'file',
 669 ##soundhw => 'string',
 670
 671 while (my ($k, $v) = each %$confdesc) {
 672     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 673 }
 674
 675 my $MAX_IDE_DISKS = 4;
 676 my $MAX_SCSI_DISKS = 14;
 677 my $MAX_VIRTIO_DISKS = 16;
 678 my $MAX_SATA_DISKS = 6;
 679 my $MAX_USB_DEVICES = 5;
 680 my $MAX_NETS = 32;
 681 my $MAX_UNUSED_DISKS = 256;
 682 my $MAX_HOSTPCI_DEVICES = 4;
 683 my $MAX_SERIAL_PORTS = 4;
 684 my $MAX_PARALLEL_PORTS = 3;
 685 my $MAX_NUMA = 8;
 686
 687 my $numa_fmt = {
 688     cpus => {
 689         type => "string",
 690         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 691         description => "CPUs accessing this NUMA node.",
 692         format_description => "id[-id];...",
 693     },
 694     memory => {
 695         type => "number",
 696         description => "Amount of memory this NUMA node provides.",
 697         optional => 1,
 698     },
 699     hostnodes => {
 700         type => "string",
 701         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 702         description => "Host NUMA nodes to use.",
 703         format_description => "id[-id];...",
 704         optional => 1,
 705     },
 706     policy => {
 707         type => 'string',
 708         enum => [qw(preferred bind interleave)],
 709         description => "NUMA allocation policy.",
 710         optional => 1,
 711     },
 712 };
 713 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 714 my $numadesc = {
 715     optional => 1,
 716     type => 'string', format => $numa_fmt,
 717     description => "NUMA topology.",
 718 };
 719 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 720
 721 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 722     $confdesc->{"numa$i"} = $numadesc;
 723 }
 724
 725 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 726                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 727                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 728 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 729
 730 my $net_fmt_bridge_descr = <<__EOD__;
 731 Bridge to attach the network device to. The Proxmox VE standard bridge
 732 is called 'vmbr0'.
 733
 734 If you do not specify a bridge, we create a kvm user (NATed) network
 735 device, which provides DHCP and DNS services. The following addresses
 736 are used:
 737
 738  10.0.2.2   Gateway
 739  10.0.2.3   DNS Server
 740  10.0.2.4   SMB Server
 741
 742 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 743 __EOD__
 744
 745 my $net_fmt = {
 746     macaddr => {
 747         type => 'string',
 748         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 749         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 750         format_description => "XX:XX:XX:XX:XX:XX",
 751         optional => 1,
 752     },
 753     model => {
 754         type => 'string',
 755         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 756         enum => $nic_model_list,
 757         default_key => 1,
 758     },
 759     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 760     bridge => {
 761         type => 'string',
 762         description => $net_fmt_bridge_descr,
 763         format_description => 'bridge',
 764         optional => 1,
 765     },
 766     queues => {
 767         type => 'integer',
 768         minimum => 0, maximum => 16,
 769         description => 'Number of packet queues to be used on the device.',
 770         optional => 1,
 771     },
 772     rate => {
 773         type => 'number',
 774         minimum => 0,
 775         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 776         optional => 1,
 777     },
 778     tag => {
 779         type => 'integer',
 780         minimum => 1, maximum => 4094,
 781         description => 'VLAN tag to apply to packets on this interface.',
 782         optional => 1,
 783     },
 784     trunks => {
 785         type => 'string',
 786         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 787         description => 'VLAN trunks to pass through this interface.',
 788         format_description => 'vlanid[;vlanid...]',
 789         optional => 1,
 790     },
 791     firewall => {
 792         type => 'boolean',
 793         description => 'Whether this interface should be protected by the firewall.',
 794         optional => 1,
 795     },
 796     link_down => {
 797         type => 'boolean',
 798         description => 'Whether this interface should be disconnected (like pulling the plug).',
 799         optional => 1,
 800     },
 801 };
 802
 803 my $netdesc = {
 804     optional => 1,
 805     type => 'string', format => $net_fmt,
 806     description => "Specify network devices.",
 807 };
 808
 809 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 810
 811 my $ipconfig_fmt = {
 812     ip => {
 813         type => 'string',
 814         format => 'pve-ipv4-config',
 815         format_description => 'IPv4Format/CIDR',
 816         description => 'IPv4 address in CIDR format.',
 817         optional => 1,
 818         default => 'dhcp',
 819     },
 820     gw => {
 821         type => 'string',
 822         format => 'ipv4',
 823         format_description => 'GatewayIPv4',
 824         description => 'Default gateway for IPv4 traffic.',
 825         optional => 1,
 826         requires => 'ip',
 827     },
 828     ip6 => {
 829         type => 'string',
 830         format => 'pve-ipv6-config',
 831         format_description => 'IPv6Format/CIDR',
 832         description => 'IPv6 address in CIDR format.',
 833         optional => 1,
 834         default => 'dhcp',
 835     },
 836     gw6 => {
 837         type => 'string',
 838         format => 'ipv6',
 839         format_description => 'GatewayIPv6',
 840         description => 'Default gateway for IPv6 traffic.',
 841         optional => 1,
 842         requires => 'ip6',
 843     },
 844 };
 845 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 846 my $ipconfigdesc = {
 847     optional => 1,
 848     type => 'string', format => 'pve-qm-ipconfig',
 849     description => <<'EODESCR',
 850 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 851
 852 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 853
 854 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 855 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 856
 857 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 858 EODESCR
 859 };
 860 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 861
 862 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 863     $confdesc->{"net$i"} = $netdesc;
 864     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 865 }
 866
 867 foreach my $key (keys %$confdesc_cloudinit) {
 868     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 869 }
 870
 871 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 872 sub verify_volume_id_or_qm_path {
 873     my ($volid, $noerr) = @_;
 874
 875     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 876         return $volid;
 877     }
 878
 879     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 880     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 881     if ($@) {
 882         return undef if $noerr;
 883         die $@;
 884     }
 885     return $volid;
 886 }
 887
 888 my $drivename_hash;
 889
 890 my %drivedesc_base = (
 891     volume => { alias => 'file' },
 892     file => {
 893         type => 'string',
 894         format => 'pve-volume-id-or-qm-path',
 895         default_key => 1,
 896         format_description => 'volume',
 897         description => "The drive's backing volume.",
 898     },
 899     media => {
 900         type => 'string',
 901         enum => [qw(cdrom disk)],
 902         description => "The drive's media type.",
 903         default => 'disk',
 904         optional => 1
 905     },
 906     cyls => {
 907         type => 'integer',
 908         description => "Force the drive's physical geometry to have a specific cylinder count.",
 909         optional => 1
 910     },
 911     heads => {
 912         type => 'integer',
 913         description => "Force the drive's physical geometry to have a specific head count.",
 914         optional => 1
 915     },
 916     secs => {
 917         type => 'integer',
 918         description => "Force the drive's physical geometry to have a specific sector count.",
 919         optional => 1
 920     },
 921     trans => {
 922         type => 'string',
 923         enum => [qw(none lba auto)],
 924         description => "Force disk geometry bios translation mode.",
 925         optional => 1,
 926     },
 927     snapshot => {
 928         type => 'boolean',
 929         description => "Controls qemu's snapshot mode feature."
 930             . " If activated, changes made to the disk are temporary and will"
 931             . " be discarded when the VM is shutdown.",
 932         optional => 1,
 933     },
 934     cache => {
 935         type => 'string',
 936         enum => [qw(none writethrough writeback unsafe directsync)],
 937         description => "The drive's cache mode",
 938         optional => 1,
 939     },
 940     format => get_standard_option('pve-qm-image-format'),
 941     size => {
 942         type => 'string',
 943         format => 'disk-size',
 944         format_description => 'DiskSize',
 945         description => "Disk size. This is purely informational and has no effect.",
 946         optional => 1,
 947     },
 948     backup => {
 949         type => 'boolean',
 950         description => "Whether the drive should be included when making backups.",
 951         optional => 1,
 952     },
 953     replicate => {
 954         type => 'boolean',
 955         description => 'Whether the drive should considered for replication jobs.',
 956         optional => 1,
 957         default => 1,
 958     },
 959     rerror => {
 960         type => 'string',
 961         enum => [qw(ignore report stop)],
 962         description => 'Read error action.',
 963         optional => 1,
 964     },
 965     werror => {
 966         type => 'string',
 967         enum => [qw(enospc ignore report stop)],
 968         description => 'Write error action.',
 969         optional => 1,
 970     },
 971     aio => {
 972         type => 'string',
 973         enum => [qw(native threads)],
 974         description => 'AIO type to use.',
 975         optional => 1,
 976     },
 977     discard => {
 978         type => 'string',
 979         enum => [qw(ignore on)],
 980         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 981         optional => 1,
 982     },
 983     detect_zeroes => {
 984         type => 'boolean',
 985         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 986         optional => 1,
 987     },
 988     serial => {
 989         type => 'string',
 990         format => 'urlencoded',
 991         format_description => 'serial',
 992         maxLength => 20*3, # *3 since it's %xx url enoded
 993         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 994         optional => 1,
 995     },
 996     shared => {
 997         type => 'boolean',
 998         description => 'Mark this locally-managed volume as available on all nodes',
 999         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1000         optional => 1,
1001         default => 0,
1002     }
1003 );
1004
1005 my %iothread_fmt = ( iothread => {
1006         type => 'boolean',
1007         description => "Whether to use iothreads for this drive",
1008         optional => 1,
1009 });
1010
1011 my %model_fmt = (
1012     model => {
1013         type => 'string',
1014         format => 'urlencoded',
1015         format_description => 'model',
1016         maxLength => 40*3, # *3 since it's %xx url enoded
1017         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1018         optional => 1,
1019     },
1020 );
1021
1022 my %queues_fmt = (
1023     queues => {
1024         type => 'integer',
1025         description => "Number of queues.",
1026         minimum => 2,
1027         optional => 1
1028     }
1029 );
1030
1031 my %scsiblock_fmt = (
1032     scsiblock => {
1033         type => 'boolean',
1034         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1035         optional => 1,
1036         default => 0,
1037     },
1038 );
1039
1040 my %ssd_fmt = (
1041     ssd => {
1042         type => 'boolean',
1043         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1044         optional => 1,
1045     },
1046 );
1047
1048 my $add_throttle_desc = sub {
1049     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1050     my $d = {
1051         type => $type,
1052         format_description => $unit,
1053         description => "Maximum $what in $longunit.",
1054         optional => 1,
1055     };
1056     $d->{minimum} = $minimum if defined($minimum);
1057     $drivedesc_base{$key} = $d;
1058 };
1059 # throughput: (leaky bucket)
1060 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1061 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1062 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1063 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1064 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1065 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1066 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1067 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1068 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1069
1070 # pools: (pool of IO before throttling starts taking effect)
1071 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1072 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1073 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1074 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1075 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1076 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1077
1078 # burst lengths
1079 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1080 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1081 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1082 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1083 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1084 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1085
1086 # legacy support
1087 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1088 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1089 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1090 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1091
1092 my $ide_fmt = {
1093     %drivedesc_base,
1094     %model_fmt,
1095     %ssd_fmt,
1096 };
1097 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1098
1099 my $idedesc = {
1100     optional => 1,
1101     type => 'string', format => $ide_fmt,
1102     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1103 };
1104 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1105
1106 my $scsi_fmt = {
1107     %drivedesc_base,
1108     %iothread_fmt,
1109     %queues_fmt,
1110     %scsiblock_fmt,
1111     %ssd_fmt,
1112 };
1113 my $scsidesc = {
1114     optional => 1,
1115     type => 'string', format => $scsi_fmt,
1116     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1117 };
1118 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1119
1120 my $sata_fmt = {
1121     %drivedesc_base,
1122     %ssd_fmt,
1123 };
1124 my $satadesc = {
1125     optional => 1,
1126     type => 'string', format => $sata_fmt,
1127     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1128 };
1129 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1130
1131 my $virtio_fmt = {
1132     %drivedesc_base,
1133     %iothread_fmt,
1134 };
1135 my $virtiodesc = {
1136     optional => 1,
1137     type => 'string', format => $virtio_fmt,
1138     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1139 };
1140 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1141
1142 my $alldrive_fmt = {
1143     %drivedesc_base,
1144     %iothread_fmt,
1145     %model_fmt,
1146     %queues_fmt,
1147     %scsiblock_fmt,
1148     %ssd_fmt,
1149 };
1150
1151 my $efidisk_fmt = {
1152     volume => { alias => 'file' },
1153     file => {
1154         type => 'string',
1155         format => 'pve-volume-id-or-qm-path',
1156         default_key => 1,
1157         format_description => 'volume',
1158         description => "The drive's backing volume.",
1159     },
1160     format => get_standard_option('pve-qm-image-format'),
1161     size => {
1162         type => 'string',
1163         format => 'disk-size',
1164         format_description => 'DiskSize',
1165         description => "Disk size. This is purely informational and has no effect.",
1166         optional => 1,
1167     },
1168 };
1169
1170 my $efidisk_desc = {
1171     optional => 1,
1172     type => 'string', format => $efidisk_fmt,
1173     description => "Configure a Disk for storing EFI vars",
1174 };
1175
1176 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1177
1178 my $usb_fmt = {
1179     host => {
1180         default_key => 1,
1181         type => 'string', format => 'pve-qm-usb-device',
1182         format_description => 'HOSTUSBDEVICE|spice',
1183         description => <<EODESCR,
1184 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1185
1186  'bus-port(.port)*' (decimal numbers) or
1187  'vendor_id:product_id' (hexadeciaml numbers) or
1188  'spice'
1189
1190 You can use the 'lsusb -t' command to list existing usb devices.
1191
1192 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1193
1194 The value 'spice' can be used to add a usb redirection devices for spice.
1195 EODESCR
1196     },
1197     usb3 => {
1198         optional => 1,
1199         type => 'boolean',
1200         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1201         default => 0,
1202     },
1203 };
1204
1205 my $usbdesc = {
1206     optional => 1,
1207     type => 'string', format => $usb_fmt,
1208     description => "Configure an USB device (n is 0 to 4).",
1209 };
1210 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1211
1212 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1213 my $hostpci_fmt = {
1214     host => {
1215         default_key => 1,
1216         type => 'string',
1217         pattern => qr/$PCIRE(;$PCIRE)*/,
1218         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1219         description => <<EODESCR,
1220 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1221 of PCI virtual functions of the host. HOSTPCIID syntax is:
1222
1223 'bus:dev.func' (hexadecimal numbers)
1224
1225 You can us the 'lspci' command to list existing PCI devices.
1226 EODESCR
1227     },
1228     rombar => {
1229         type => 'boolean',
1230         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1231         optional => 1,
1232         default => 1,
1233     },
1234     romfile => {
1235         type => 'string',
1236         pattern => '[^,;]+',
1237         format_description => 'string',
1238         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1239         optional => 1,
1240     },
1241     pcie => {
1242         type => 'boolean',
1243         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1244         optional => 1,
1245         default => 0,
1246     },
1247     'x-vga' => {
1248         type => 'boolean',
1249         description =>  "Enable vfio-vga device support.",
1250         optional => 1,
1251         default => 0,
1252     },
1253     'mdev' => {
1254         type => 'string',
1255         format_description => 'string',
1256         pattern => '[^/\.:]+',
1257         optional => 1,
1258         description => <<EODESCR
1259 The type of mediated device to use.
1260 An instance of this type will be created on startup of the VM and
1261 will be cleaned up when the VM stops.
1262 EODESCR
1263     }
1264 };
1265 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1266
1267 my $hostpcidesc = {
1268         optional => 1,
1269         type => 'string', format => 'pve-qm-hostpci',
1270         description => "Map host PCI devices into guest.",
1271         verbose_description =>  <<EODESCR,
1272 Map host PCI devices into guest.
1273
1274 NOTE: This option allows direct access to host hardware. So it is no longer
1275 possible to migrate such machines - use with special care.
1276
1277 CAUTION: Experimental! User reported problems with this option.
1278 EODESCR
1279 };
1280 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1281
1282 my $serialdesc = {
1283         optional => 1,
1284         type => 'string',
1285         pattern => '(/dev/.+|socket)',
1286         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1287         verbose_description =>  <<EODESCR,
1288 Create a serial device inside the VM (n is 0 to 3), and pass through a
1289 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1290 host side (use 'qm terminal' to open a terminal connection).
1291
1292 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1293
1294 CAUTION: Experimental! User reported problems with this option.
1295 EODESCR
1296 };
1297
1298 my $paralleldesc= {
1299         optional => 1,
1300         type => 'string',
1301         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1302         description =>  "Map host parallel devices (n is 0 to 2).",
1303         verbose_description =>  <<EODESCR,
1304 Map host parallel devices (n is 0 to 2).
1305
1306 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1307
1308 CAUTION: Experimental! User reported problems with this option.
1309 EODESCR
1310 };
1311
1312 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1313     $confdesc->{"parallel$i"} = $paralleldesc;
1314 }
1315
1316 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1317     $confdesc->{"serial$i"} = $serialdesc;
1318 }
1319
1320 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1321     $confdesc->{"hostpci$i"} = $hostpcidesc;
1322 }
1323
1324 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1325     $drivename_hash->{"ide$i"} = 1;
1326     $confdesc->{"ide$i"} = $idedesc;
1327 }
1328
1329 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1330     $drivename_hash->{"sata$i"} = 1;
1331     $confdesc->{"sata$i"} = $satadesc;
1332 }
1333
1334 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1335     $drivename_hash->{"scsi$i"} = 1;
1336     $confdesc->{"scsi$i"} = $scsidesc ;
1337 }
1338
1339 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1340     $drivename_hash->{"virtio$i"} = 1;
1341     $confdesc->{"virtio$i"} = $virtiodesc;
1342 }
1343
1344 $drivename_hash->{efidisk0} = 1;
1345 $confdesc->{efidisk0} = $efidisk_desc;
1346
1347 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1348     $confdesc->{"usb$i"} = $usbdesc;
1349 }
1350
1351 my $unuseddesc = {
1352     optional => 1,
1353     type => 'string', format => 'pve-volume-id',
1354     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1355 };
1356
1357 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1358     $confdesc->{"unused$i"} = $unuseddesc;
1359 }
1360
1361 my $kvm_api_version = 0;
1362
1363 sub kvm_version {
1364     return $kvm_api_version if $kvm_api_version;
1365
1366     open my $fh, '<', '/dev/kvm'
1367         or return undef;
1368
1369     # 0xae00 => KVM_GET_API_VERSION
1370     $kvm_api_version = ioctl($fh, 0xae00, 0);
1371
1372     return $kvm_api_version;
1373 }
1374
1375 my $kvm_user_version;
1376
1377 sub kvm_user_version {
1378
1379     return $kvm_user_version if $kvm_user_version;
1380
1381     $kvm_user_version = 'unknown';
1382
1383     my $code = sub {
1384         my $line = shift;
1385         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1386             $kvm_user_version = $2;
1387         }
1388     };
1389
1390     eval { run_command("kvm -version", outfunc => $code); };
1391     warn $@ if $@;
1392
1393     return $kvm_user_version;
1394
1395 }
1396
1397 sub kernel_has_vhost_net {
1398     return -c '/dev/vhost-net';
1399 }
1400
1401 sub valid_drive_names {
1402     # order is important - used to autoselect boot disk
1403     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1404             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1405             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1406             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1407             'efidisk0');
1408 }
1409
1410 sub is_valid_drivename {
1411     my $dev = shift;
1412
1413     return defined($drivename_hash->{$dev});
1414 }
1415
1416 sub option_exists {
1417     my $key = shift;
1418     return defined($confdesc->{$key});
1419 }
1420
1421 sub nic_models {
1422     return $nic_model_list;
1423 }
1424
1425 sub os_list_description {
1426
1427     return {
1428         other => 'Other',
1429         wxp => 'Windows XP',
1430         w2k => 'Windows 2000',
1431         w2k3 =>, 'Windows 2003',
1432         w2k8 => 'Windows 2008',
1433         wvista => 'Windows Vista',
1434         win7 => 'Windows 7',
1435         win8 => 'Windows 8/2012',
1436         win10 => 'Windows 10/2016',
1437         l24 => 'Linux 2.4',
1438         l26 => 'Linux 2.6',
1439     };
1440 }
1441
1442 my $cdrom_path;
1443
1444 sub get_cdrom_path {
1445
1446     return  $cdrom_path if $cdrom_path;
1447
1448     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1449     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1450     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1451 }
1452
1453 sub get_iso_path {
1454     my ($storecfg, $vmid, $cdrom) = @_;
1455
1456     if ($cdrom eq 'cdrom') {
1457         return get_cdrom_path();
1458     } elsif ($cdrom eq 'none') {
1459         return '';
1460     } elsif ($cdrom =~ m|^/|) {
1461         return $cdrom;
1462     } else {
1463         return PVE::Storage::path($storecfg, $cdrom);
1464     }
1465 }
1466
1467 # try to convert old style file names to volume IDs
1468 sub filename_to_volume_id {
1469     my ($vmid, $file, $media) = @_;
1470
1471      if (!($file eq 'none' || $file eq 'cdrom' ||
1472           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1473
1474         return undef if $file =~ m|/|;
1475
1476         if ($media && $media eq 'cdrom') {
1477             $file = "local:iso/$file";
1478         } else {
1479             $file = "local:$vmid/$file";
1480         }
1481     }
1482
1483     return $file;
1484 }
1485
1486 sub verify_media_type {
1487     my ($opt, $vtype, $media) = @_;
1488
1489     return if !$media;
1490
1491     my $etype;
1492     if ($media eq 'disk') {
1493         $etype = 'images';
1494     } elsif ($media eq 'cdrom') {
1495         $etype = 'iso';
1496     } else {
1497         die "internal error";
1498     }
1499
1500     return if ($vtype eq $etype);
1501
1502     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1503 }
1504
1505 sub cleanup_drive_path {
1506     my ($opt, $storecfg, $drive) = @_;
1507
1508     # try to convert filesystem paths to volume IDs
1509
1510     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1511         ($drive->{file} !~ m|^/dev/.+|) &&
1512         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1513         ($drive->{file} !~ m/^\d+$/)) {
1514         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1515         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1516         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1517         verify_media_type($opt, $vtype, $drive->{media});
1518         $drive->{file} = $volid;
1519     }
1520
1521     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1522 }
1523
1524 sub parse_hotplug_features {
1525     my ($data) = @_;
1526
1527     my $res = {};
1528
1529     return $res if $data eq '0';
1530
1531     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1532
1533     foreach my $feature (PVE::Tools::split_list($data)) {
1534         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1535             $res->{$1} = 1;
1536         } else {
1537             die "invalid hotplug feature '$feature'\n";
1538         }
1539     }
1540     return $res;
1541 }
1542
1543 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1544 sub pve_verify_hotplug_features {
1545     my ($value, $noerr) = @_;
1546
1547     return $value if parse_hotplug_features($value);
1548
1549     return undef if $noerr;
1550
1551     die "unable to parse hotplug option\n";
1552 }
1553
1554 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1555 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1556 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1557 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1558 #        [,iothread=on][,serial=serial][,model=model]
1559
1560 sub parse_drive {
1561     my ($key, $data) = @_;
1562
1563     my ($interface, $index);
1564
1565     if ($key =~ m/^([^\d]+)(\d+)$/) {
1566         $interface = $1;
1567         $index = $2;
1568     } else {
1569         return undef;
1570     }
1571
1572     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1573                                      : $confdesc->{$key}->{format};
1574     if (!$desc) {
1575         warn "invalid drive key: $key\n";
1576         return undef;
1577     }
1578     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1579     return undef if !$res;
1580     $res->{interface} = $interface;
1581     $res->{index} = $index;
1582
1583     my $error = 0;
1584     foreach my $opt (qw(bps bps_rd bps_wr)) {
1585         if (my $bps = defined(delete $res->{$opt})) {
1586             if (defined($res->{"m$opt"})) {
1587                 warn "both $opt and m$opt specified\n";
1588                 ++$error;
1589                 next;
1590             }
1591             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1592         }
1593     }
1594
1595     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1596     for my $requirement (
1597         [mbps_max => 'mbps'],
1598         [mbps_rd_max => 'mbps_rd'],
1599         [mbps_wr_max => 'mbps_wr'],
1600         [miops_max => 'miops'],
1601         [miops_rd_max => 'miops_rd'],
1602         [miops_wr_max => 'miops_wr'],
1603         [bps_max_length => 'mbps_max'],
1604         [bps_rd_max_length => 'mbps_rd_max'],
1605         [bps_wr_max_length => 'mbps_wr_max'],
1606         [iops_max_length => 'iops_max'],
1607         [iops_rd_max_length => 'iops_rd_max'],
1608         [iops_wr_max_length => 'iops_wr_max']) {
1609         my ($option, $requires) = @$requirement;
1610         if ($res->{$option} && !$res->{$requires}) {
1611             warn "$option requires $requires\n";
1612             ++$error;
1613         }
1614     }
1615
1616     return undef if $error;
1617
1618     return undef if $res->{mbps_rd} && $res->{mbps};
1619     return undef if $res->{mbps_wr} && $res->{mbps};
1620     return undef if $res->{iops_rd} && $res->{iops};
1621     return undef if $res->{iops_wr} && $res->{iops};
1622
1623     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1624         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1625         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1626         return undef if $res->{interface} eq 'virtio';
1627     }
1628
1629     if (my $size = $res->{size}) {
1630         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1631     }
1632
1633     return $res;
1634 }
1635
1636 sub print_drive {
1637     my ($vmid, $drive) = @_;
1638     my $data = { %$drive };
1639     delete $data->{$_} for qw(index interface);
1640     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1641 }
1642
1643 sub scsi_inquiry {
1644     my($fh, $noerr) = @_;
1645
1646     my $SG_IO = 0x2285;
1647     my $SG_GET_VERSION_NUM = 0x2282;
1648
1649     my $versionbuf = "\x00" x 8;
1650     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1651     if (!$ret) {
1652         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1653         return undef;
1654     }
1655     my $version = unpack("I", $versionbuf);
1656     if ($version < 30000) {
1657         die "scsi generic interface too old\n"  if !$noerr;
1658         return undef;
1659     }
1660
1661     my $buf = "\x00" x 36;
1662     my $sensebuf = "\x00" x 8;
1663     my $cmd = pack("C x3 C x1", 0x12, 36);
1664
1665     # see /usr/include/scsi/sg.h
1666     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1667
1668     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1669                       length($sensebuf), 0, length($buf), $buf,
1670                       $cmd, $sensebuf, 6000);
1671
1672     $ret = ioctl($fh, $SG_IO, $packet);
1673     if (!$ret) {
1674         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1675         return undef;
1676     }
1677
1678     my @res = unpack($sg_io_hdr_t, $packet);
1679     if ($res[17] || $res[18]) {
1680         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1681         return undef;
1682     }
1683
1684     my $res = {};
1685     (my $byte0, my $byte1, $res->{vendor},
1686      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1687
1688     $res->{removable} = $byte1 & 128 ? 1 : 0;
1689     $res->{type} = $byte0 & 31;
1690
1691     return $res;
1692 }
1693
1694 sub path_is_scsi {
1695     my ($path) = @_;
1696
1697     my $fh = IO::File->new("+<$path") || return undef;
1698     my $res = scsi_inquiry($fh, 1);
1699     close($fh);
1700
1701     return $res;
1702 }
1703
1704 sub machine_type_is_q35 {
1705     my ($conf) = @_;
1706
1707     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1708 }
1709
1710 sub print_tabletdevice_full {
1711     my ($conf, $arch) = @_;
1712
1713     my $q35 = machine_type_is_q35($conf);
1714
1715     # we use uhci for old VMs because tablet driver was buggy in older qemu
1716     my $usbbus;
1717     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1718         $usbbus = 'ehci';
1719     } else {
1720         $usbbus = 'uhci';
1721     }
1722
1723     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1724 }
1725
1726 sub print_keyboarddevice_full {
1727     my ($conf, $arch, $machine) = @_;
1728
1729     return undef if $arch ne 'aarch64';
1730
1731     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1732 }
1733
1734 sub print_drivedevice_full {
1735     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1736
1737     my $device = '';
1738     my $maxdev = 0;
1739
1740     if ($drive->{interface} eq 'virtio') {
1741         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1742         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1743         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1744     } elsif ($drive->{interface} eq 'scsi') {
1745
1746         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1747         my $unit = $drive->{index} % $maxdev;
1748         my $devicetype = 'hd';
1749         my $path = '';
1750         if (drive_is_cdrom($drive)) {
1751             $devicetype = 'cd';
1752         } else {
1753             if ($drive->{file} =~ m|^/|) {
1754                 $path = $drive->{file};
1755                 if (my $info = path_is_scsi($path)) {
1756                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1757                         $devicetype = 'block';
1758                     } elsif ($info->{type} == 1) { # tape
1759                         $devicetype = 'generic';
1760                     }
1761                 }
1762             } else {
1763                  $path = PVE::Storage::path($storecfg, $drive->{file});
1764             }
1765
1766             if($path =~ m/^iscsi\:\/\//){
1767                 $devicetype = 'generic';
1768             }
1769         }
1770
1771         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1772             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1773         } else {
1774             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1775         }
1776
1777         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1778             $device .= ",rotation_rate=1";
1779         }
1780
1781     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1782         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1783         my $controller = int($drive->{index} / $maxdev);
1784         my $unit = $drive->{index} % $maxdev;
1785         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1786
1787         $device = "ide-$devicetype";
1788         if ($drive->{interface} eq 'ide') {
1789             $device .= ",bus=ide.$controller,unit=$unit";
1790         } else {
1791             $device .= ",bus=ahci$controller.$unit";
1792         }
1793         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1794
1795         if ($devicetype eq 'hd') {
1796             if (my $model = $drive->{model}) {
1797                 $model = URI::Escape::uri_unescape($model);
1798                 $device .= ",model=$model";
1799             }
1800             if ($drive->{ssd}) {
1801                 $device .= ",rotation_rate=1";
1802             }
1803         }
1804     } elsif ($drive->{interface} eq 'usb') {
1805         die "implement me";
1806         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1807     } else {
1808         die "unsupported interface type";
1809     }
1810
1811     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1812
1813     if (my $serial = $drive->{serial}) {
1814         $serial = URI::Escape::uri_unescape($serial);
1815         $device .= ",serial=$serial";
1816     }
1817
1818
1819     return $device;
1820 }
1821
1822 sub get_initiator_name {
1823     my $initiator;
1824
1825     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1826     while (defined(my $line = <$fh>)) {
1827         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1828         $initiator = $1;
1829         last;
1830     }
1831     $fh->close();
1832
1833     return $initiator;
1834 }
1835
1836 sub print_drive_full {
1837     my ($storecfg, $vmid, $drive) = @_;
1838
1839     my $path;
1840     my $volid = $drive->{file};
1841     my $format;
1842
1843     if (drive_is_cdrom($drive)) {
1844         $path = get_iso_path($storecfg, $vmid, $volid);
1845     } else {
1846         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1847         if ($storeid) {
1848             $path = PVE::Storage::path($storecfg, $volid);
1849             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1850             $format = qemu_img_format($scfg, $volname);
1851         } else {
1852             $path = $volid;
1853             $format = "raw";
1854         }
1855    }
1856
1857     my $opts = '';
1858     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1859     foreach my $o (@qemu_drive_options) {
1860         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1861     }
1862
1863     # snapshot only accepts on|off
1864     if (defined($drive->{snapshot})) {
1865         my $v = $drive->{snapshot} ? 'on' : 'off';
1866         $opts .= ",snapshot=$v";
1867     }
1868
1869     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1870         my ($dir, $qmpname) = @$type;
1871         if (my $v = $drive->{"mbps$dir"}) {
1872             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1873         }
1874         if (my $v = $drive->{"mbps${dir}_max"}) {
1875             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1876         }
1877         if (my $v = $drive->{"bps${dir}_max_length"}) {
1878             $opts .= ",throttling.bps$qmpname-max-length=$v";
1879         }
1880         if (my $v = $drive->{"iops${dir}"}) {
1881             $opts .= ",throttling.iops$qmpname=$v";
1882         }
1883         if (my $v = $drive->{"iops${dir}_max"}) {
1884             $opts .= ",throttling.iops$qmpname-max=$v";
1885         }
1886         if (my $v = $drive->{"iops${dir}_max_length"}) {
1887             $opts .= ",throttling.iops$qmpname-max-length=$v";
1888         }
1889     }
1890
1891     $opts .= ",format=$format" if $format && !$drive->{format};
1892
1893     my $cache_direct = 0;
1894
1895     if (my $cache = $drive->{cache}) {
1896         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1897     } elsif (!drive_is_cdrom($drive)) {
1898         $opts .= ",cache=none";
1899         $cache_direct = 1;
1900     }
1901
1902     # aio native works only with O_DIRECT
1903     if (!$drive->{aio}) {
1904         if($cache_direct) {
1905             $opts .= ",aio=native";
1906         } else {
1907             $opts .= ",aio=threads";
1908         }
1909     }
1910
1911     if (!drive_is_cdrom($drive)) {
1912         my $detectzeroes;
1913         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1914             $detectzeroes = 'off';
1915         } elsif ($drive->{discard}) {
1916             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1917         } else {
1918             # This used to be our default with discard not being specified:
1919             $detectzeroes = 'on';
1920         }
1921         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1922     }
1923
1924     my $pathinfo = $path ? "file=$path," : '';
1925
1926     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1927 }
1928
1929 sub print_netdevice_full {
1930     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1931
1932     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1933
1934     my $device = $net->{model};
1935     if ($net->{model} eq 'virtio') {
1936          $device = 'virtio-net-pci';
1937      };
1938
1939     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1940     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1941     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1942         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1943         my $vectors = $net->{queues} * 2 + 2;
1944         $tmpstr .= ",vectors=$vectors,mq=on";
1945     }
1946     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1947
1948     if ($use_old_bios_files) {
1949         my $romfile;
1950         if ($device eq 'virtio-net-pci') {
1951             $romfile = 'pxe-virtio.rom';
1952         } elsif ($device eq 'e1000') {
1953             $romfile = 'pxe-e1000.rom';
1954         } elsif ($device eq 'ne2k') {
1955             $romfile = 'pxe-ne2k_pci.rom';
1956         } elsif ($device eq 'pcnet') {
1957             $romfile = 'pxe-pcnet.rom';
1958         } elsif ($device eq 'rtl8139') {
1959             $romfile = 'pxe-rtl8139.rom';
1960         }
1961         $tmpstr .= ",romfile=$romfile" if $romfile;
1962     }
1963
1964     return $tmpstr;
1965 }
1966
1967 sub print_netdev_full {
1968     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1969
1970     my $i = '';
1971     if ($netid =~ m/^net(\d+)$/) {
1972         $i = int($1);
1973     }
1974
1975     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1976
1977     my $ifname = "tap${vmid}i$i";
1978
1979     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1980     die "interface name '$ifname' is too long (max 15 character)\n"
1981         if length($ifname) >= 16;
1982
1983     my $vhostparam = '';
1984     if (is_native($arch)) {
1985         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1986     }
1987
1988     my $vmname = $conf->{name} || "vm$vmid";
1989
1990     my $netdev = "";
1991     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1992
1993     if ($net->{bridge}) {
1994         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1995     } else {
1996         $netdev = "type=user,id=$netid,hostname=$vmname";
1997     }
1998
1999     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2000
2001     return $netdev;
2002 }
2003
2004
2005 sub print_cpu_device {
2006     my ($conf, $id) = @_;
2007
2008     my $kvm = $conf->{kvm} // 1;
2009     my $cpu = $kvm ? "kvm64" : "qemu64";
2010     if (my $cputype = $conf->{cpu}) {
2011         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2012             or die "Cannot parse cpu description: $cputype\n";
2013         $cpu = $cpuconf->{cputype};
2014     }
2015
2016     my $cores = $conf->{cores} || 1;
2017
2018     my $current_core = ($id - 1) % $cores;
2019     my $current_socket = int(($id - 1 - $current_core)/$cores);
2020
2021     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2022 }
2023
2024 my $vga_map = {
2025     'cirrus' => 'cirrus-vga',
2026     'std' => 'VGA',
2027     'vmware' => 'vmware-svga',
2028     'virtio' => 'virtio-vga',
2029 };
2030
2031 sub print_vga_device {
2032     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2033
2034     my $type = $vga_map->{$vga->{type}};
2035     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2036         $type = 'virtio-gpu';
2037     }
2038     my $vgamem_mb = $vga->{memory};
2039     if ($qxlnum) {
2040         $type = $id ? 'qxl' : 'qxl-vga';
2041     }
2042     die "no devicetype for $vga->{type}\n" if !$type;
2043
2044     my $memory = "";
2045     if ($vgamem_mb) {
2046         if ($vga->{type} eq 'virtio') {
2047             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2048             $memory = ",max_hostmem=$bytes";
2049         } elsif ($qxlnum) {
2050             # from https://www.spice-space.org/multiple-monitors.html
2051             $memory = ",vgamem_mb=$vga->{memory}";
2052             my $ram = $vgamem_mb * 4;
2053             my $vram = $vgamem_mb * 2;
2054             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2055         } else {
2056             $memory = ",vgamem_mb=$vga->{memory}";
2057         }
2058     } elsif ($qxlnum && $id) {
2059         $memory = ",ram_size=67108864,vram_size=33554432";
2060     }
2061
2062     my $q35 = machine_type_is_q35($conf);
2063     my $vgaid = "vga" . ($id // '');
2064     my $pciaddr;
2065
2066     if ($q35 && $vgaid eq 'vga') {
2067         # the first display uses pcie.0 bus on q35 machines
2068         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2069     } else {
2070         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2071     }
2072
2073     return "$type,id=${vgaid}${memory}${pciaddr}";
2074 }
2075
2076 sub drive_is_cloudinit {
2077     my ($drive) = @_;
2078     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2079 }
2080
2081 sub drive_is_cdrom {
2082     my ($drive, $exclude_cloudinit) = @_;
2083
2084     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2085
2086     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2087
2088 }
2089
2090 sub parse_number_sets {
2091     my ($set) = @_;
2092     my $res = [];
2093     foreach my $part (split(/;/, $set)) {
2094         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2095             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2096             push @$res, [ $1, $2 ];
2097         } else {
2098             die "invalid range: $part\n";
2099         }
2100     }
2101     return $res;
2102 }
2103
2104 sub parse_numa {
2105     my ($data) = @_;
2106
2107     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2108     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2109     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2110     return $res;
2111 }
2112
2113 sub parse_hostpci {
2114     my ($value) = @_;
2115
2116     return undef if !$value;
2117
2118     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2119
2120     my @idlist = split(/;/, $res->{host});
2121     delete $res->{host};
2122     foreach my $id (@idlist) {
2123         if ($id =~ m/\./) { # full id 00:00.1
2124             push @{$res->{pciid}}, {
2125                 id => $id,
2126             };
2127         } else { # partial id 00:00
2128             $res->{pciid} = PVE::SysFSTools::lspci($id);
2129         }
2130     }
2131     return $res;
2132 }
2133
2134 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2135 sub parse_net {
2136     my ($data) = @_;
2137
2138     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2139     if ($@) {
2140         warn $@;
2141         return undef;
2142     }
2143     if (!defined($res->{macaddr})) {
2144         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2145         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2146     }
2147     return $res;
2148 }
2149
2150 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2151 sub parse_ipconfig {
2152     my ($data) = @_;
2153
2154     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2155     if ($@) {
2156         warn $@;
2157         return undef;
2158     }
2159
2160     if ($res->{gw} && !$res->{ip}) {
2161         warn 'gateway specified without specifying an IP address';
2162         return undef;
2163     }
2164     if ($res->{gw6} && !$res->{ip6}) {
2165         warn 'IPv6 gateway specified without specifying an IPv6 address';
2166         return undef;
2167     }
2168     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2169         warn 'gateway specified together with DHCP';
2170         return undef;
2171     }
2172     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2173         # gw6 + auto/dhcp
2174         warn "IPv6 gateway specified together with $res->{ip6} address";
2175         return undef;
2176     }
2177
2178     if (!$res->{ip} && !$res->{ip6}) {
2179         return { ip => 'dhcp', ip6 => 'dhcp' };
2180     }
2181
2182     return $res;
2183 }
2184
2185 sub print_net {
2186     my $net = shift;
2187
2188     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2189 }
2190
2191 sub add_random_macs {
2192     my ($settings) = @_;
2193
2194     foreach my $opt (keys %$settings) {
2195         next if $opt !~ m/^net(\d+)$/;
2196         my $net = parse_net($settings->{$opt});
2197         next if !$net;
2198         $settings->{$opt} = print_net($net);
2199     }
2200 }
2201
2202 sub vm_is_volid_owner {
2203     my ($storecfg, $vmid, $volid) = @_;
2204
2205     if ($volid !~  m|^/|) {
2206         my ($path, $owner);
2207         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2208         if ($owner && ($owner == $vmid)) {
2209             return 1;
2210         }
2211     }
2212
2213     return undef;
2214 }
2215
2216 sub split_flagged_list {
2217     my $text = shift || '';
2218     $text =~ s/[,;]/ /g;
2219     $text =~ s/^\s+//;
2220     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2221 }
2222
2223 sub join_flagged_list {
2224     my ($how, $lst) = @_;
2225     join $how, map { $lst->{$_} . $_ } keys %$lst;
2226 }
2227
2228 sub vmconfig_delete_pending_option {
2229     my ($conf, $key, $force) = @_;
2230
2231     delete $conf->{pending}->{$key};
2232     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2233     $pending_delete_hash->{$key} = $force ? '!' : '';
2234     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2235 }
2236
2237 sub vmconfig_undelete_pending_option {
2238     my ($conf, $key) = @_;
2239
2240     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2241     delete $pending_delete_hash->{$key};
2242
2243     if (%$pending_delete_hash) {
2244         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2245     } else {
2246         delete $conf->{pending}->{delete};
2247     }
2248 }
2249
2250 sub vmconfig_register_unused_drive {
2251     my ($storecfg, $vmid, $conf, $drive) = @_;
2252
2253     if (drive_is_cloudinit($drive)) {
2254         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2255         warn $@ if $@;
2256     } elsif (!drive_is_cdrom($drive)) {
2257         my $volid = $drive->{file};
2258         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2259             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2260         }
2261     }
2262 }
2263
2264 sub vmconfig_cleanup_pending {
2265     my ($conf) = @_;
2266
2267     # remove pending changes when nothing changed
2268     my $changes;
2269     foreach my $opt (keys %{$conf->{pending}}) {
2270         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2271             $changes = 1;
2272             delete $conf->{pending}->{$opt};
2273         }
2274     }
2275
2276     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2277     my $pending_delete_hash = {};
2278     while (my ($opt, $force) = each %$current_delete_hash) {
2279         if (defined($conf->{$opt})) {
2280             $pending_delete_hash->{$opt} = $force;
2281         } else {
2282             $changes = 1;
2283         }
2284     }
2285
2286     if (%$pending_delete_hash) {
2287         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2288     } else {
2289         delete $conf->{pending}->{delete};
2290     }
2291
2292     return $changes;
2293 }
2294
2295 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2296 my $smbios1_fmt = {
2297     uuid => {
2298         type => 'string',
2299         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2300         format_description => 'UUID',
2301         description => "Set SMBIOS1 UUID.",
2302         optional => 1,
2303     },
2304     version => {
2305         type => 'string',
2306         pattern => '\S+',
2307         format_description => 'string',
2308         description => "Set SMBIOS1 version.",
2309         optional => 1,
2310     },
2311     serial => {
2312         type => 'string',
2313         pattern => '\S+',
2314         format_description => 'string',
2315         description => "Set SMBIOS1 serial number.",
2316         optional => 1,
2317     },
2318     manufacturer => {
2319         type => 'string',
2320         pattern => '\S+',
2321         format_description => 'string',
2322         description => "Set SMBIOS1 manufacturer.",
2323         optional => 1,
2324     },
2325     product => {
2326         type => 'string',
2327         pattern => '\S+',
2328         format_description => 'string',
2329         description => "Set SMBIOS1 product ID.",
2330         optional => 1,
2331     },
2332     sku => {
2333         type => 'string',
2334         pattern => '\S+',
2335         format_description => 'string',
2336         description => "Set SMBIOS1 SKU string.",
2337         optional => 1,
2338     },
2339     family => {
2340         type => 'string',
2341         pattern => '\S+',
2342         format_description => 'string',
2343         description => "Set SMBIOS1 family string.",
2344         optional => 1,
2345     },
2346 };
2347
2348 sub parse_smbios1 {
2349     my ($data) = @_;
2350
2351     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2352     warn $@ if $@;
2353     return $res;
2354 }
2355
2356 sub print_smbios1 {
2357     my ($smbios1) = @_;
2358     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2359 }
2360
2361 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2362
2363 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2364 sub verify_bootdisk {
2365     my ($value, $noerr) = @_;
2366
2367     return $value if is_valid_drivename($value);
2368
2369     return undef if $noerr;
2370
2371     die "invalid boot disk '$value'\n";
2372 }
2373
2374 sub parse_watchdog {
2375     my ($value) = @_;
2376
2377     return undef if !$value;
2378
2379     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2380     warn $@ if $@;
2381     return $res;
2382 }
2383
2384 sub parse_guest_agent {
2385     my ($value) = @_;
2386
2387     return {} if !defined($value->{agent});
2388
2389     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2390     warn $@ if $@;
2391
2392     # if the agent is disabled ignore the other potentially set properties
2393     return {} if !$res->{enabled};
2394     return $res;
2395 }
2396
2397 sub parse_vga {
2398     my ($value) = @_;
2399
2400     return {} if !$value;
2401     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2402     warn $@ if $@;
2403     return $res;
2404 }
2405
2406 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2407 sub verify_usb_device {
2408     my ($value, $noerr) = @_;
2409
2410     return $value if parse_usb_device($value);
2411
2412     return undef if $noerr;
2413
2414     die "unable to parse usb device\n";
2415 }
2416
2417 # add JSON properties for create and set function
2418 sub json_config_properties {
2419     my $prop = shift;
2420
2421     foreach my $opt (keys %$confdesc) {
2422         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2423         $prop->{$opt} = $confdesc->{$opt};
2424     }
2425
2426     return $prop;
2427 }
2428
2429 # return copy of $confdesc_cloudinit to generate documentation
2430 sub cloudinit_config_properties {
2431
2432     return dclone($confdesc_cloudinit);
2433 }
2434
2435 sub check_type {
2436     my ($key, $value) = @_;
2437
2438     die "unknown setting '$key'\n" if !$confdesc->{$key};
2439
2440     my $type = $confdesc->{$key}->{type};
2441
2442     if (!defined($value)) {
2443         die "got undefined value\n";
2444     }
2445
2446     if ($value =~ m/[\n\r]/) {
2447         die "property contains a line feed\n";
2448     }
2449
2450     if ($type eq 'boolean') {
2451         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2452         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2453         die "type check ('boolean') failed - got '$value'\n";
2454     } elsif ($type eq 'integer') {
2455         return int($1) if $value =~ m/^(\d+)$/;
2456         die "type check ('integer') failed - got '$value'\n";
2457     } elsif ($type eq 'number') {
2458         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2459         die "type check ('number') failed - got '$value'\n";
2460     } elsif ($type eq 'string') {
2461         if (my $fmt = $confdesc->{$key}->{format}) {
2462             PVE::JSONSchema::check_format($fmt, $value);
2463             return $value;
2464         }
2465         $value =~ s/^\"(.*)\"$/$1/;
2466         return $value;
2467     } else {
2468         die "internal error"
2469     }
2470 }
2471
2472 sub touch_config {
2473     my ($vmid) = @_;
2474
2475     my $conf = PVE::QemuConfig->config_file($vmid);
2476     utime undef, undef, $conf;
2477 }
2478
2479 sub destroy_vm {
2480     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2481
2482     my $conffile = PVE::QemuConfig->config_file($vmid);
2483
2484     my $conf = PVE::QemuConfig->load_config($vmid);
2485
2486     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2487
2488     if ($conf->{template}) {
2489         # check if any base image is still used by a linked clone
2490         foreach_drive($conf, sub {
2491                 my ($ds, $drive) = @_;
2492
2493                 return if drive_is_cdrom($drive);
2494
2495                 my $volid = $drive->{file};
2496
2497                 return if !$volid || $volid =~ m|^/|;
2498
2499                 die "base volume '$volid' is still in use by linked cloned\n"
2500                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2501
2502         });
2503     }
2504
2505     # only remove disks owned by this VM
2506     foreach_drive($conf, sub {
2507         my ($ds, $drive) = @_;
2508
2509         return if drive_is_cdrom($drive, 1);
2510
2511         my $volid = $drive->{file};
2512
2513         return if !$volid || $volid =~ m|^/|;
2514
2515         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2516         return if !$path || !$owner || ($owner != $vmid);
2517
2518         eval {
2519             PVE::Storage::vdisk_free($storecfg, $volid);
2520         };
2521         warn "Could not remove disk '$volid', check manually: $@" if $@;
2522
2523     });
2524
2525     if ($keep_empty_config) {
2526         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2527     } else {
2528         unlink $conffile;
2529     }
2530
2531     # also remove unused disk
2532     eval {
2533         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2534
2535         eval {
2536             PVE::Storage::foreach_volid($dl, sub {
2537                 my ($volid, $sid, $volname, $d) = @_;
2538                 PVE::Storage::vdisk_free($storecfg, $volid);
2539             });
2540         };
2541         warn $@ if $@;
2542
2543     };
2544     warn $@ if $@;
2545 }
2546
2547 sub parse_vm_config {
2548     my ($filename, $raw) = @_;
2549
2550     return undef if !defined($raw);
2551
2552     my $res = {
2553         digest => Digest::SHA::sha1_hex($raw),
2554         snapshots => {},
2555         pending => {},
2556     };
2557
2558     $filename =~ m|/qemu-server/(\d+)\.conf$|
2559         || die "got strange filename '$filename'";
2560
2561     my $vmid = $1;
2562
2563     my $conf = $res;
2564     my $descr;
2565     my $section = '';
2566
2567     my @lines = split(/\n/, $raw);
2568     foreach my $line (@lines) {
2569         next if $line =~ m/^\s*$/;
2570
2571         if ($line =~ m/^\[PENDING\]\s*$/i) {
2572             $section = 'pending';
2573             if (defined($descr)) {
2574                 $descr =~ s/\s+$//;
2575                 $conf->{description} = $descr;
2576             }
2577             $descr = undef;
2578             $conf = $res->{$section} = {};
2579             next;
2580
2581         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2582             $section = $1;
2583             if (defined($descr)) {
2584                 $descr =~ s/\s+$//;
2585                 $conf->{description} = $descr;
2586             }
2587             $descr = undef;
2588             $conf = $res->{snapshots}->{$section} = {};
2589             next;
2590         }
2591
2592         if ($line =~ m/^\#(.*)\s*$/) {
2593             $descr = '' if !defined($descr);
2594             $descr .= PVE::Tools::decode_text($1) . "\n";
2595             next;
2596         }
2597
2598         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2599             $descr = '' if !defined($descr);
2600             $descr .= PVE::Tools::decode_text($2);
2601         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2602             $conf->{snapstate} = $1;
2603         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2604             my $key = $1;
2605             my $value = $2;
2606             $conf->{$key} = $value;
2607         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2608             my $value = $1;
2609             if ($section eq 'pending') {
2610                 $conf->{delete} = $value; # we parse this later
2611             } else {
2612                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2613             }
2614         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2615             my $key = $1;
2616             my $value = $2;
2617             eval { $value = check_type($key, $value); };
2618             if ($@) {
2619                 warn "vm $vmid - unable to parse value of '$key' - $@";
2620             } else {
2621                 $key = 'ide2' if $key eq 'cdrom';
2622                 my $fmt = $confdesc->{$key}->{format};
2623                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2624                     my $v = parse_drive($key, $value);
2625                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2626                         $v->{file} = $volid;
2627                         $value = print_drive($vmid, $v);
2628                     } else {
2629                         warn "vm $vmid - unable to parse value of '$key'\n";
2630                         next;
2631                     }
2632                 }
2633
2634                 $conf->{$key} = $value;
2635             }
2636         }
2637     }
2638
2639     if (defined($descr)) {
2640         $descr =~ s/\s+$//;
2641         $conf->{description} = $descr;
2642     }
2643     delete $res->{snapstate}; # just to be sure
2644
2645     return $res;
2646 }
2647
2648 sub write_vm_config {
2649     my ($filename, $conf) = @_;
2650
2651     delete $conf->{snapstate}; # just to be sure
2652
2653     if ($conf->{cdrom}) {
2654         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2655         $conf->{ide2} = $conf->{cdrom};
2656         delete $conf->{cdrom};
2657     }
2658
2659     # we do not use 'smp' any longer
2660     if ($conf->{sockets}) {
2661         delete $conf->{smp};
2662     } elsif ($conf->{smp}) {
2663         $conf->{sockets} = $conf->{smp};
2664         delete $conf->{cores};
2665         delete $conf->{smp};
2666     }
2667
2668     my $used_volids = {};
2669
2670     my $cleanup_config = sub {
2671         my ($cref, $pending, $snapname) = @_;
2672
2673         foreach my $key (keys %$cref) {
2674             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2675                 $key eq 'snapstate' || $key eq 'pending';
2676             my $value = $cref->{$key};
2677             if ($key eq 'delete') {
2678                 die "propertry 'delete' is only allowed in [PENDING]\n"
2679                     if !$pending;
2680                 # fixme: check syntax?
2681                 next;
2682             }
2683             eval { $value = check_type($key, $value); };
2684             die "unable to parse value of '$key' - $@" if $@;
2685
2686             $cref->{$key} = $value;
2687
2688             if (!$snapname && is_valid_drivename($key)) {
2689                 my $drive = parse_drive($key, $value);
2690                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2691             }
2692         }
2693     };
2694
2695     &$cleanup_config($conf);
2696
2697     &$cleanup_config($conf->{pending}, 1);
2698
2699     foreach my $snapname (keys %{$conf->{snapshots}}) {
2700         die "internal error" if $snapname eq 'pending';
2701         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2702     }
2703
2704     # remove 'unusedX' settings if we re-add a volume
2705     foreach my $key (keys %$conf) {
2706         my $value = $conf->{$key};
2707         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2708             delete $conf->{$key};
2709         }
2710     }
2711
2712     my $generate_raw_config = sub {
2713         my ($conf, $pending) = @_;
2714
2715         my $raw = '';
2716
2717         # add description as comment to top of file
2718         if (defined(my $descr = $conf->{description})) {
2719             if ($descr) {
2720                 foreach my $cl (split(/\n/, $descr)) {
2721                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2722                 }
2723             } else {
2724                 $raw .= "#\n" if $pending;
2725             }
2726         }
2727
2728         foreach my $key (sort keys %$conf) {
2729             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2730             $raw .= "$key: $conf->{$key}\n";
2731         }
2732         return $raw;
2733     };
2734
2735     my $raw = &$generate_raw_config($conf);
2736
2737     if (scalar(keys %{$conf->{pending}})){
2738         $raw .= "\n[PENDING]\n";
2739         $raw .= &$generate_raw_config($conf->{pending}, 1);
2740     }
2741
2742     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2743         $raw .= "\n[$snapname]\n";
2744         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2745     }
2746
2747     return $raw;
2748 }
2749
2750 sub load_defaults {
2751
2752     my $res = {};
2753
2754     # we use static defaults from our JSON schema configuration
2755     foreach my $key (keys %$confdesc) {
2756         if (defined(my $default = $confdesc->{$key}->{default})) {
2757             $res->{$key} = $default;
2758         }
2759     }
2760
2761     return $res;
2762 }
2763
2764 sub config_list {
2765     my $vmlist = PVE::Cluster::get_vmlist();
2766     my $res = {};
2767     return $res if !$vmlist || !$vmlist->{ids};
2768     my $ids = $vmlist->{ids};
2769
2770     foreach my $vmid (keys %$ids) {
2771         my $d = $ids->{$vmid};
2772         next if !$d->{node} || $d->{node} ne $nodename;
2773         next if !$d->{type} || $d->{type} ne 'qemu';
2774         $res->{$vmid}->{exists} = 1;
2775     }
2776     return $res;
2777 }
2778
2779 # test if VM uses local resources (to prevent migration)
2780 sub check_local_resources {
2781     my ($conf, $noerr) = @_;
2782
2783     my $loc_res = 0;
2784
2785     $loc_res = 1 if $conf->{hostusb}; # old syntax
2786     $loc_res = 1 if $conf->{hostpci}; # old syntax
2787
2788     foreach my $k (keys %$conf) {
2789         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2790         # sockets are safe: they will recreated be on the target side post-migrate
2791         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2792         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2793     }
2794
2795     die "VM uses local resources\n" if $loc_res && !$noerr;
2796
2797     return $loc_res;
2798 }
2799
2800 # check if used storages are available on all nodes (use by migrate)
2801 sub check_storage_availability {
2802     my ($storecfg, $conf, $node) = @_;
2803
2804     foreach_drive($conf, sub {
2805         my ($ds, $drive) = @_;
2806
2807         my $volid = $drive->{file};
2808         return if !$volid;
2809
2810         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2811         return if !$sid;
2812
2813         # check if storage is available on both nodes
2814         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2815         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2816    });
2817 }
2818
2819 # list nodes where all VM images are available (used by has_feature API)
2820 sub shared_nodes {
2821     my ($conf, $storecfg) = @_;
2822
2823     my $nodelist = PVE::Cluster::get_nodelist();
2824     my $nodehash = { map { $_ => 1 } @$nodelist };
2825     my $nodename = PVE::INotify::nodename();
2826
2827     foreach_drive($conf, sub {
2828         my ($ds, $drive) = @_;
2829
2830         my $volid = $drive->{file};
2831         return if !$volid;
2832
2833         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2834         if ($storeid) {
2835             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2836             if ($scfg->{disable}) {
2837                 $nodehash = {};
2838             } elsif (my $avail = $scfg->{nodes}) {
2839                 foreach my $node (keys %$nodehash) {
2840                     delete $nodehash->{$node} if !$avail->{$node};
2841                 }
2842             } elsif (!$scfg->{shared}) {
2843                 foreach my $node (keys %$nodehash) {
2844                     delete $nodehash->{$node} if $node ne $nodename
2845                 }
2846             }
2847         }
2848     });
2849
2850     return $nodehash
2851 }
2852
2853 sub check_cmdline {
2854     my ($pidfile, $pid) = @_;
2855
2856     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2857     if (defined($fh)) {
2858         my $line = <$fh>;
2859         $fh->close;
2860         return undef if !$line;
2861         my @param = split(/\0/, $line);
2862
2863         my $cmd = $param[0];
2864         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2865
2866         for (my $i = 0; $i < scalar (@param); $i++) {
2867             my $p = $param[$i];
2868             next if !$p;
2869             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2870                 my $p = $param[$i+1];
2871                 return 1 if $p && ($p eq $pidfile);
2872                 return undef;
2873             }
2874         }
2875     }
2876     return undef;
2877 }
2878
2879 sub check_running {
2880     my ($vmid, $nocheck, $node) = @_;
2881
2882     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2883
2884     die "unable to find configuration file for VM $vmid - no such machine\n"
2885         if !$nocheck && ! -f $filename;
2886
2887     my $pidfile = pidfile_name($vmid);
2888
2889     if (my $fd = IO::File->new("<$pidfile")) {
2890         my $st = stat($fd);
2891         my $line = <$fd>;
2892         close($fd);
2893
2894         my $mtime = $st->mtime;
2895         if ($mtime > time()) {
2896             warn "file '$filename' modified in future\n";
2897         }
2898
2899         if ($line =~ m/^(\d+)$/) {
2900             my $pid = $1;
2901             if (check_cmdline($pidfile, $pid)) {
2902                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2903                     return $pid;
2904                 }
2905             }
2906         }
2907     }
2908
2909     return undef;
2910 }
2911
2912 sub vzlist {
2913
2914     my $vzlist = config_list();
2915
2916     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2917
2918     while (defined(my $de = $fd->read)) {
2919         next if $de !~ m/^(\d+)\.pid$/;
2920         my $vmid = $1;
2921         next if !defined($vzlist->{$vmid});
2922         if (my $pid = check_running($vmid)) {
2923             $vzlist->{$vmid}->{pid} = $pid;
2924         }
2925     }
2926
2927     return $vzlist;
2928 }
2929
2930 sub disksize {
2931     my ($storecfg, $conf) = @_;
2932
2933     my $bootdisk = $conf->{bootdisk};
2934     return undef if !$bootdisk;
2935     return undef if !is_valid_drivename($bootdisk);
2936
2937     return undef if !$conf->{$bootdisk};
2938
2939     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2940     return undef if !defined($drive);
2941
2942     return undef if drive_is_cdrom($drive);
2943
2944     my $volid = $drive->{file};
2945     return undef if !$volid;
2946
2947     return $drive->{size};
2948 }
2949
2950 our $vmstatus_return_properties = {
2951     vmid => get_standard_option('pve-vmid'),
2952     status => {
2953         description => "Qemu process status.",
2954         type => 'string',
2955         enum => ['stopped', 'running'],
2956     },
2957     maxmem => {
2958         description => "Maximum memory in bytes.",
2959         type => 'integer',
2960         optional => 1,
2961         renderer => 'bytes',
2962     },
2963     maxdisk => {
2964         description => "Root disk size in bytes.",
2965         type => 'integer',
2966         optional => 1,
2967         renderer => 'bytes',
2968     },
2969     name => {
2970         description => "VM name.",
2971         type => 'string',
2972         optional => 1,
2973     },
2974     qmpstatus => {
2975         description => "Qemu QMP agent status.",
2976         type => 'string',
2977         optional => 1,
2978     },
2979     pid => {
2980         description => "PID of running qemu process.",
2981         type => 'integer',
2982         optional => 1,
2983     },
2984     uptime => {
2985         description => "Uptime.",
2986         type => 'integer',
2987         optional => 1,
2988         renderer => 'duration',
2989     },
2990     cpus => {
2991         description => "Maximum usable CPUs.",
2992         type => 'number',
2993         optional => 1,
2994     },
2995 };
2996
2997 my $last_proc_pid_stat;
2998
2999 # get VM status information
3000 # This must be fast and should not block ($full == false)
3001 # We only query KVM using QMP if $full == true (this can be slow)
3002 sub vmstatus {
3003     my ($opt_vmid, $full) = @_;
3004
3005     my $res = {};
3006
3007     my $storecfg = PVE::Storage::config();
3008
3009     my $list = vzlist();
3010     my $defaults = load_defaults();
3011
3012     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3013
3014     my $cpucount = $cpuinfo->{cpus} || 1;
3015
3016     foreach my $vmid (keys %$list) {
3017         next if $opt_vmid && ($vmid ne $opt_vmid);
3018
3019         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3020         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3021
3022         my $d = { vmid => $vmid };
3023         $d->{pid} = $list->{$vmid}->{pid};
3024
3025         # fixme: better status?
3026         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3027
3028         my $size = disksize($storecfg, $conf);
3029         if (defined($size)) {
3030             $d->{disk} = 0; # no info available
3031             $d->{maxdisk} = $size;
3032         } else {
3033             $d->{disk} = 0;
3034             $d->{maxdisk} = 0;
3035         }
3036
3037         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3038             * ($conf->{cores} || $defaults->{cores});
3039         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3040         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3041
3042         $d->{name} = $conf->{name} || "VM $vmid";
3043         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3044             : $defaults->{memory}*(1024*1024);
3045
3046         if ($conf->{balloon}) {
3047             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3048             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3049                 : $defaults->{shares};
3050         }
3051
3052         $d->{uptime} = 0;
3053         $d->{cpu} = 0;
3054         $d->{mem} = 0;
3055
3056         $d->{netout} = 0;
3057         $d->{netin} = 0;
3058
3059         $d->{diskread} = 0;
3060         $d->{diskwrite} = 0;
3061
3062         $d->{template} = PVE::QemuConfig->is_template($conf);
3063
3064         $d->{serial} = 1 if conf_has_serial($conf);
3065
3066         $res->{$vmid} = $d;
3067     }
3068
3069     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3070     foreach my $dev (keys %$netdev) {
3071         next if $dev !~ m/^tap([1-9]\d*)i/;
3072         my $vmid = $1;
3073         my $d = $res->{$vmid};
3074         next if !$d;
3075
3076         $d->{netout} += $netdev->{$dev}->{receive};
3077         $d->{netin} += $netdev->{$dev}->{transmit};
3078
3079         if ($full) {
3080             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3081             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3082         }
3083
3084     }
3085
3086     my $ctime = gettimeofday;
3087
3088     foreach my $vmid (keys %$list) {
3089
3090         my $d = $res->{$vmid};
3091         my $pid = $d->{pid};
3092         next if !$pid;
3093
3094         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3095         next if !$pstat; # not running
3096
3097         my $used = $pstat->{utime} + $pstat->{stime};
3098
3099         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3100
3101         if ($pstat->{vsize}) {
3102             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3103         }
3104
3105         my $old = $last_proc_pid_stat->{$pid};
3106         if (!$old) {
3107             $last_proc_pid_stat->{$pid} = {
3108                 time => $ctime,
3109                 used => $used,
3110                 cpu => 0,
3111             };
3112             next;
3113         }
3114
3115         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3116
3117         if ($dtime > 1000) {
3118             my $dutime = $used -  $old->{used};
3119
3120             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3121             $last_proc_pid_stat->{$pid} = {
3122                 time => $ctime,
3123                 used => $used,
3124                 cpu => $d->{cpu},
3125             };
3126         } else {
3127             $d->{cpu} = $old->{cpu};
3128         }
3129     }
3130
3131     return $res if !$full;
3132
3133     my $qmpclient = PVE::QMPClient->new();
3134
3135     my $ballooncb = sub {
3136         my ($vmid, $resp) = @_;
3137
3138         my $info = $resp->{'return'};
3139         return if !$info->{max_mem};
3140
3141         my $d = $res->{$vmid};
3142
3143         # use memory assigned to VM
3144         $d->{maxmem} = $info->{max_mem};
3145         $d->{balloon} = $info->{actual};
3146
3147         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3148             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3149             $d->{freemem} = $info->{free_mem};
3150         }
3151
3152         $d->{ballooninfo} = $info;
3153     };
3154
3155     my $blockstatscb = sub {
3156         my ($vmid, $resp) = @_;
3157         my $data = $resp->{'return'} || [];
3158         my $totalrdbytes = 0;
3159         my $totalwrbytes = 0;
3160
3161         for my $blockstat (@$data) {
3162             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3163             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3164
3165             $blockstat->{device} =~ s/drive-//;
3166             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3167         }
3168         $res->{$vmid}->{diskread} = $totalrdbytes;
3169         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3170     };
3171
3172     my $statuscb = sub {
3173         my ($vmid, $resp) = @_;
3174
3175         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3176         # this fails if ballon driver is not loaded, so this must be
3177         # the last commnand (following command are aborted if this fails).
3178         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3179
3180         my $status = 'unknown';
3181         if (!defined($status = $resp->{'return'}->{status})) {
3182             warn "unable to get VM status\n";
3183             return;
3184         }
3185
3186         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3187     };
3188
3189     foreach my $vmid (keys %$list) {
3190         next if $opt_vmid && ($vmid ne $opt_vmid);
3191         next if !$res->{$vmid}->{pid}; # not running
3192         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3193     }
3194
3195     $qmpclient->queue_execute(undef, 2);
3196
3197     foreach my $vmid (keys %$list) {
3198         next if $opt_vmid && ($vmid ne $opt_vmid);
3199         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3200     }
3201
3202     return $res;
3203 }
3204
3205 sub foreach_drive {
3206     my ($conf, $func, @param) = @_;
3207
3208     foreach my $ds (valid_drive_names()) {
3209         next if !defined($conf->{$ds});
3210
3211         my $drive = parse_drive($ds, $conf->{$ds});
3212         next if !$drive;
3213
3214         &$func($ds, $drive, @param);
3215     }
3216 }
3217
3218 sub foreach_volid {
3219     my ($conf, $func, @param) = @_;
3220
3221     my $volhash = {};
3222
3223     my $test_volid = sub {
3224         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3225
3226         return if !$volid;
3227
3228         $volhash->{$volid}->{cdrom} //= 1;
3229         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3230
3231         $volhash->{$volid}->{replicate} //= 0;
3232         $volhash->{$volid}->{replicate} = 1 if $replicate;
3233
3234         $volhash->{$volid}->{shared} //= 0;
3235         $volhash->{$volid}->{shared} = 1 if $shared;
3236
3237         $volhash->{$volid}->{referenced_in_config} //= 0;
3238         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3239
3240         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3241             if defined($snapname);
3242     };
3243
3244     foreach_drive($conf, sub {
3245         my ($ds, $drive) = @_;
3246         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3247     });
3248
3249     foreach my $snapname (keys %{$conf->{snapshots}}) {
3250         my $snap = $conf->{snapshots}->{$snapname};
3251         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3252         foreach_drive($snap, sub {
3253             my ($ds, $drive) = @_;
3254             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3255         });
3256     }
3257
3258     foreach my $volid (keys %$volhash) {
3259         &$func($volid, $volhash->{$volid}, @param);
3260     }
3261 }
3262
3263 sub conf_has_serial {
3264     my ($conf) = @_;
3265
3266     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3267         if ($conf->{"serial$i"}) {
3268             return 1;
3269         }
3270     }
3271
3272     return 0;
3273 }
3274
3275 sub vga_conf_has_spice {
3276     my ($vga) = @_;
3277
3278     my $vgaconf = parse_vga($vga);
3279     my $vgatype = $vgaconf->{type};
3280     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3281
3282     return $1 || 1;
3283 }
3284
3285 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3286 sub get_host_arch() {
3287     $host_arch = (POSIX::uname())[4] if !$host_arch;
3288     return $host_arch;
3289 }
3290
3291 sub is_native($) {
3292     my ($arch) = @_;
3293     return get_host_arch() eq $arch;
3294 }
3295
3296 my $default_machines = {
3297     x86_64 => 'pc',
3298     aarch64 => 'virt',
3299 };
3300
3301 sub get_basic_machine_info {
3302     my ($conf, $forcemachine) = @_;
3303
3304     my $arch = $conf->{arch} // get_host_arch();
3305     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3306     return ($arch, $machine);
3307 }
3308
3309 sub get_ovmf_files($) {
3310     my ($arch) = @_;
3311
3312     my $ovmf = $OVMF->{$arch}
3313         or die "no OVMF images known for architecture '$arch'\n";
3314
3315     return @$ovmf;
3316 }
3317
3318 my $Arch2Qemu = {
3319     aarch64 => '/usr/bin/qemu-system-aarch64',
3320     x86_64 => '/usr/bin/qemu-system-x86_64',
3321 };
3322 sub get_command_for_arch($) {
3323     my ($arch) = @_;
3324     return '/usr/bin/kvm' if is_native($arch);
3325
3326     my $cmd = $Arch2Qemu->{$arch}
3327         or die "don't know how to emulate architecture '$arch'\n";
3328     return $cmd;
3329 }
3330
3331 sub get_cpu_options {
3332     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3333
3334     my $cpuFlags = [];
3335     my $ostype = $conf->{ostype};
3336
3337     my $cpu = $kvm ? "kvm64" : "qemu64";
3338     if ($arch eq 'aarch64') {
3339         $cpu = 'cortex-a57';
3340     }
3341     my $hv_vendor_id;
3342     if (my $cputype = $conf->{cpu}) {
3343         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3344             or die "Cannot parse cpu description: $cputype\n";
3345         $cpu = $cpuconf->{cputype};
3346         $kvm_off = 1 if $cpuconf->{hidden};
3347         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3348
3349         if (defined(my $flags = $cpuconf->{flags})) {
3350             push @$cpuFlags, split(";", $flags);
3351         }
3352     }
3353
3354     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3355
3356     push @$cpuFlags , '-x2apic'
3357         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3358
3359     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3360
3361     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3362
3363     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3364
3365         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3366         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3367     }
3368
3369     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3370
3371     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3372
3373     push @$cpuFlags, 'kvm=off' if $kvm_off;
3374
3375     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3376         push @$cpuFlags, "vendor=${cpu_vendor}"
3377             if $cpu_vendor ne 'default';
3378     } elsif ($arch ne 'aarch64') {
3379         die "internal error"; # should not happen
3380     }
3381
3382     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3383
3384     return ('-cpu', $cpu);
3385 }
3386
3387 sub config_to_command {
3388     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3389
3390     my $cmd = [];
3391     my $globalFlags = [];
3392     my $machineFlags = [];
3393     my $rtcFlags = [];
3394     my $devices = [];
3395     my $pciaddr = '';
3396     my $bridges = {};
3397     my $kvmver = kvm_user_version();
3398     my $vernum = 0; # unknown
3399     my $ostype = $conf->{ostype};
3400     my $winversion = windows_version($ostype);
3401     my $kvm = $conf->{kvm};
3402
3403     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3404     $kvm //= 1 if is_native($arch);
3405
3406     if ($kvm) {
3407         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3408             if !defined kvm_version();
3409     }
3410
3411     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3412         $vernum = $1*1000000+$2*1000;
3413     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3414         $vernum = $1*1000000+$2*1000+$3;
3415     }
3416
3417     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3418
3419     my $have_ovz = -f '/proc/vz/vestat';
3420
3421     my $q35 = machine_type_is_q35($conf);
3422     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3423     my $use_old_bios_files = undef;
3424     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3425
3426     my $cpuunits = defined($conf->{cpuunits}) ?
3427             $conf->{cpuunits} : $defaults->{cpuunits};
3428
3429     push @$cmd, get_command_for_arch($arch);
3430
3431     push @$cmd, '-id', $vmid;
3432
3433     my $vmname = $conf->{name} || "vm$vmid";
3434
3435     push @$cmd, '-name', $vmname;
3436
3437     my $use_virtio = 0;
3438
3439     my $qmpsocket = qmp_socket($vmid);
3440     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3441     push @$cmd, '-mon', "chardev=qmp,mode=control";
3442
3443     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3444         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3445         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3446     }
3447
3448     push @$cmd, '-pidfile' , pidfile_name($vmid);
3449
3450     push @$cmd, '-daemonize';
3451
3452     if ($conf->{smbios1}) {
3453         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3454     }
3455
3456     if ($conf->{vmgenid}) {
3457         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3458     }
3459
3460     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3461     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3462         die "uefi base image not found\n" if ! -f $ovmf_code;
3463
3464         my $path;
3465         my $format;
3466         if (my $efidisk = $conf->{efidisk0}) {
3467             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3468             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3469             $format = $d->{format};
3470             if ($storeid) {
3471                 $path = PVE::Storage::path($storecfg, $d->{file});
3472                 if (!defined($format)) {
3473                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3474                     $format = qemu_img_format($scfg, $volname);
3475                 }
3476             } else {
3477                 $path = $d->{file};
3478                 die "efidisk format must be specified\n"
3479                     if !defined($format);
3480             }
3481         } else {
3482             warn "no efidisk configured! Using temporary efivars disk.\n";
3483             $path = "/tmp/$vmid-ovmf.fd";
3484             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3485             $format = 'raw';
3486         }
3487
3488         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3489         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3490     }
3491
3492
3493     # add usb controllers
3494     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3495     push @$devices, @usbcontrollers if @usbcontrollers;
3496     my $vga = parse_vga($conf->{vga});
3497
3498     my $qxlnum = vga_conf_has_spice($conf->{vga});
3499     $vga->{type} = 'qxl' if $qxlnum;
3500
3501     if (!$vga->{type}) {
3502         if ($arch eq 'aarch64') {
3503             $vga->{type} = 'virtio';
3504         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3505             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3506         } else {
3507             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3508         }
3509     }
3510
3511     # enable absolute mouse coordinates (needed by vnc)
3512     my $tablet;
3513     if (defined($conf->{tablet})) {
3514         $tablet = $conf->{tablet};
3515     } else {
3516         $tablet = $defaults->{tablet};
3517         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3518         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3519     }
3520
3521     if ($tablet) {
3522         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3523         my $kbd = print_keyboarddevice_full($conf, $arch);
3524         push @$devices, '-device', $kbd if defined($kbd);
3525     }
3526
3527     my $kvm_off = 0;
3528     my $gpu_passthrough;
3529
3530     # host pci devices
3531     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3532         my $d = parse_hostpci($conf->{"hostpci$i"});
3533         next if !$d;
3534
3535         my $pcie = $d->{pcie};
3536         if($pcie){
3537             die "q35 machine model is not enabled" if !$q35;
3538             # win7 wants to have the pcie devices directly on the pcie bus
3539             # instead of in the root port
3540             if ($winversion == 7) {
3541                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3542             } else {
3543                 $pciaddr = print_pcie_addr("hostpci$i");
3544             }
3545         }else{
3546             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3547         }
3548
3549         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3550         my $romfile = $d->{romfile};
3551
3552         my $xvga = '';
3553         if ($d->{'x-vga'}) {
3554             $xvga = ',x-vga=on';
3555             $kvm_off = 1;
3556             $vga->{type} = 'none' if !defined($conf->{vga});
3557             $gpu_passthrough = 1;
3558
3559             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3560                 $xvga = "";
3561             }
3562         }
3563         my $pcidevices = $d->{pciid};
3564         my $multifunction = 1 if @$pcidevices > 1;
3565         my $sysfspath;
3566         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3567             my $id = $pcidevices->[0]->{id};
3568             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3569             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3570         } elsif ($d->{mdev}) {
3571             warn "ignoring mediated device with multifunction device\n";
3572         }
3573
3574         my $j=0;
3575         foreach my $pcidevice (@$pcidevices) {
3576
3577             my $id = "hostpci$i";
3578             $id .= ".$j" if $multifunction;
3579             my $addr = $pciaddr;
3580             $addr .= ".$j" if $multifunction;
3581             my $devicestr = "vfio-pci";
3582             if ($sysfspath) {
3583                 $devicestr .= ",sysfsdev=$sysfspath";
3584             } else {
3585                 $devicestr .= ",host=$pcidevice->{id}";
3586             }
3587             $devicestr .= ",id=$id$addr";
3588
3589             if($j == 0){
3590                 $devicestr .= "$rombar$xvga";
3591                 $devicestr .= ",multifunction=on" if $multifunction;
3592                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3593             }
3594
3595             push @$devices, '-device', $devicestr;
3596             $j++;
3597         }
3598     }
3599
3600     # usb devices
3601     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3602     push @$devices, @usbdevices if @usbdevices;
3603     # serial devices
3604     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3605         if (my $path = $conf->{"serial$i"}) {
3606             if ($path eq 'socket') {
3607                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3608                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3609                 # On aarch64, serial0 is the UART device. Qemu only allows
3610                 # connecting UART devices via the '-serial' command line, as
3611                 # the device has a fixed slot on the hardware...
3612                 if ($arch eq 'aarch64' && $i == 0) {
3613                     push @$devices, '-serial', "chardev:serial$i";
3614                 } else {
3615                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3616                 }
3617             } else {
3618                 die "no such serial device\n" if ! -c $path;
3619                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3620                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3621             }
3622         }
3623     }
3624
3625     # parallel devices
3626     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3627         if (my $path = $conf->{"parallel$i"}) {
3628             die "no such parallel device\n" if ! -c $path;
3629             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3630             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3631             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3632         }
3633     }
3634
3635
3636     my $sockets = 1;
3637     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3638     $sockets = $conf->{sockets} if  $conf->{sockets};
3639
3640     my $cores = $conf->{cores} || 1;
3641
3642     my $maxcpus = $sockets * $cores;
3643
3644     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3645
3646     my $allowed_vcpus = $cpuinfo->{cpus};
3647
3648     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3649         if ($allowed_vcpus < $maxcpus);
3650
3651     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3652
3653         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3654         for (my $i = 2; $i <= $vcpus; $i++)  {
3655             my $cpustr = print_cpu_device($conf,$i);
3656             push @$cmd, '-device', $cpustr;
3657         }
3658
3659     } else {
3660
3661         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3662     }
3663     push @$cmd, '-nodefaults';
3664
3665     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3666
3667     my $bootindex_hash = {};
3668     my $i = 1;
3669     foreach my $o (split(//, $bootorder)) {
3670         $bootindex_hash->{$o} = $i*100;
3671         $i++;
3672     }
3673
3674     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3675
3676     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3677
3678     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3679
3680     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3681         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3682         my $socket = vnc_socket($vmid);
3683         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3684     } else {
3685         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3686         push @$cmd, '-nographic';
3687     }
3688
3689     # time drift fix
3690     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3691
3692     my $useLocaltime = $conf->{localtime};
3693
3694     if ($winversion >= 5) { # windows
3695         $useLocaltime = 1 if !defined($conf->{localtime});
3696
3697         # use time drift fix when acpi is enabled
3698         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3699             $tdf = 1 if !defined($conf->{tdf});
3700         }
3701     }
3702
3703     if ($winversion >= 6) {
3704         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3705         push @$cmd, '-no-hpet';
3706     }
3707
3708     push @$rtcFlags, 'driftfix=slew' if $tdf;
3709
3710     if (!$kvm) {
3711         push @$machineFlags, 'accel=tcg';
3712     }
3713
3714     if ($machine_type) {
3715         push @$machineFlags, "type=${machine_type}";
3716     }
3717
3718     if ($conf->{startdate}) {
3719         push @$rtcFlags, "base=$conf->{startdate}";
3720     } elsif ($useLocaltime) {
3721         push @$rtcFlags, 'base=localtime';
3722     }
3723
3724     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3725
3726     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3727
3728     push @$cmd, '-S' if $conf->{freeze};
3729
3730     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3731
3732     # enable sound
3733     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3734     #push @$cmd, '-soundhw', 'es1370';
3735     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3736
3737     if (parse_guest_agent($conf)->{enabled}) {
3738         my $qgasocket = qmp_socket($vmid, 1);
3739         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3740         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3741         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3742         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3743     }
3744
3745     my $spice_port;
3746
3747     if ($qxlnum) {
3748         if ($qxlnum > 1) {
3749             if ($winversion){
3750                 for(my $i = 1; $i < $qxlnum; $i++){
3751                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3752                 }
3753             } else {
3754                 # assume other OS works like Linux
3755                 my ($ram, $vram) = ("134217728", "67108864");
3756                 if ($vga->{memory}) {
3757                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3758                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3759                 }
3760                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3761                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3762             }
3763         }
3764
3765         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3766
3767         my $nodename = PVE::INotify::nodename();
3768         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3769         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3770         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3771         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3772         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3773
3774         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3775
3776         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3777         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3778         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3779     }
3780
3781     # enable balloon by default, unless explicitly disabled
3782     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3783         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3784         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3785     }
3786
3787     if ($conf->{watchdog}) {
3788         my $wdopts = parse_watchdog($conf->{watchdog});
3789         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3790         my $watchdog = $wdopts->{model} || 'i6300esb';
3791         push @$devices, '-device', "$watchdog$pciaddr";
3792         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3793     }
3794
3795     my $vollist = [];
3796     my $scsicontroller = {};
3797     my $ahcicontroller = {};
3798     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3799
3800     # Add iscsi initiator name if available
3801     if (my $initiator = get_initiator_name()) {
3802         push @$devices, '-iscsi', "initiator-name=$initiator";
3803     }
3804
3805     foreach_drive($conf, sub {
3806         my ($ds, $drive) = @_;
3807
3808         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3809             push @$vollist, $drive->{file};
3810         }
3811
3812         # ignore efidisk here, already added in bios/fw handling code above
3813         return if $drive->{interface} eq 'efidisk';
3814
3815         $use_virtio = 1 if $ds =~ m/^virtio/;
3816
3817         if (drive_is_cdrom ($drive)) {
3818             if ($bootindex_hash->{d}) {
3819                 $drive->{bootindex} = $bootindex_hash->{d};
3820                 $bootindex_hash->{d} += 1;
3821             }
3822         } else {
3823             if ($bootindex_hash->{c}) {
3824                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3825                 $bootindex_hash->{c} += 1;
3826             }
3827         }
3828
3829         if($drive->{interface} eq 'virtio'){
3830            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3831         }
3832
3833         if ($drive->{interface} eq 'scsi') {
3834
3835             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3836
3837             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3838             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3839
3840             my $iothread = '';
3841             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3842                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3843                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3844             } elsif ($drive->{iothread}) {
3845                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3846             }
3847
3848             my $queues = '';
3849             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3850                 $queues = ",num_queues=$drive->{queues}";
3851             }
3852
3853             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3854             $scsicontroller->{$controller}=1;
3855         }
3856
3857         if ($drive->{interface} eq 'sata') {
3858            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3859            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3860            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3861            $ahcicontroller->{$controller}=1;
3862         }
3863
3864         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3865         push @$devices, '-drive',$drive_cmd;
3866         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3867     });
3868
3869     for (my $i = 0; $i < $MAX_NETS; $i++) {
3870          next if !$conf->{"net$i"};
3871          my $d = parse_net($conf->{"net$i"});
3872          next if !$d;
3873
3874          $use_virtio = 1 if $d->{model} eq 'virtio';
3875
3876          if ($bootindex_hash->{n}) {
3877             $d->{bootindex} = $bootindex_hash->{n};
3878             $bootindex_hash->{n} += 1;
3879          }
3880
3881          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3882          push @$devices, '-netdev', $netdevfull;
3883
3884          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3885          push @$devices, '-device', $netdevicefull;
3886     }
3887
3888     if (!$q35) {
3889         # add pci bridges
3890         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3891            $bridges->{1} = 1;
3892            $bridges->{2} = 1;
3893         }
3894
3895         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3896
3897         while (my ($k, $v) = each %$bridges) {
3898             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3899             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3900         }
3901     }
3902
3903     push @$cmd, @$devices;
3904     push @$cmd, '-rtc', join(',', @$rtcFlags)
3905         if scalar(@$rtcFlags);
3906     push @$cmd, '-machine', join(',', @$machineFlags)
3907         if scalar(@$machineFlags);
3908     push @$cmd, '-global', join(',', @$globalFlags)
3909         if scalar(@$globalFlags);
3910
3911     # add custom args
3912     if ($conf->{args}) {
3913         my $aa = PVE::Tools::split_args($conf->{args});
3914         push @$cmd, @$aa;
3915     }
3916
3917     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3918 }
3919
3920 sub vnc_socket {
3921     my ($vmid) = @_;
3922     return "${var_run_tmpdir}/$vmid.vnc";
3923 }
3924
3925 sub spice_port {
3926     my ($vmid) = @_;
3927
3928     my $res = vm_mon_cmd($vmid, 'query-spice');
3929
3930     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3931 }
3932
3933 sub qmp_socket {
3934     my ($vmid, $qga, $name) = @_;
3935     my $sockettype = $qga ? 'qga' : 'qmp';
3936     my $ext = $name ? '-'.$name : '';
3937     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3938 }
3939
3940 sub pidfile_name {
3941     my ($vmid) = @_;
3942     return "${var_run_tmpdir}/$vmid.pid";
3943 }
3944
3945 sub vm_devices_list {
3946     my ($vmid) = @_;
3947
3948     my $res = vm_mon_cmd($vmid, 'query-pci');
3949     my $devices_to_check = [];
3950     my $devices = {};
3951     foreach my $pcibus (@$res) {
3952         push @$devices_to_check, @{$pcibus->{devices}},
3953     }
3954
3955     while (@$devices_to_check) {
3956         my $to_check = [];
3957         for my $d (@$devices_to_check) {
3958             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3959             next if !$d->{'pci_bridge'};
3960
3961             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3962             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3963         }
3964         $devices_to_check = $to_check;
3965     }
3966
3967     my $resblock = vm_mon_cmd($vmid, 'query-block');
3968     foreach my $block (@$resblock) {
3969         if($block->{device} =~ m/^drive-(\S+)/){
3970                 $devices->{$1} = 1;
3971         }
3972     }
3973
3974     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3975     foreach my $mice (@$resmice) {
3976         if ($mice->{name} eq 'QEMU HID Tablet') {
3977             $devices->{tablet} = 1;
3978             last;
3979         }
3980     }
3981
3982     # for usb devices there is no query-usb
3983     # but we can iterate over the entries in
3984     # qom-list path=/machine/peripheral
3985     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3986     foreach my $per (@$resperipheral) {
3987         if ($per->{name} =~ m/^usb\d+$/) {
3988             $devices->{$per->{name}} = 1;
3989         }
3990     }
3991
3992     return $devices;
3993 }
3994
3995 sub vm_deviceplug {
3996     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3997
3998     my $q35 = machine_type_is_q35($conf);
3999
4000     my $devices_list = vm_devices_list($vmid);
4001     return 1 if defined($devices_list->{$deviceid});
4002
4003     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4004
4005     if ($deviceid eq 'tablet') {
4006
4007         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4008
4009     } elsif ($deviceid eq 'keyboard') {
4010
4011         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4012
4013     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4014
4015         die "usb hotplug currently not reliable\n";
4016         # since we can't reliably hot unplug all added usb devices
4017         # and usb passthrough disables live migration
4018         # we disable usb hotplugging for now
4019         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4020
4021     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4022
4023         qemu_iothread_add($vmid, $deviceid, $device);
4024
4025         qemu_driveadd($storecfg, $vmid, $device);
4026         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4027
4028         qemu_deviceadd($vmid, $devicefull);
4029         eval { qemu_deviceaddverify($vmid, $deviceid); };
4030         if (my $err = $@) {
4031             eval { qemu_drivedel($vmid, $deviceid); };
4032             warn $@ if $@;
4033             die $err;
4034         }
4035
4036     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4037
4038
4039         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4040         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4041         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4042
4043         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4044
4045         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4046             qemu_iothread_add($vmid, $deviceid, $device);
4047             $devicefull .= ",iothread=iothread-$deviceid";
4048         }
4049
4050         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4051             $devicefull .= ",num_queues=$device->{queues}";
4052         }
4053
4054         qemu_deviceadd($vmid, $devicefull);
4055         qemu_deviceaddverify($vmid, $deviceid);
4056
4057     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4058
4059         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4060         qemu_driveadd($storecfg, $vmid, $device);
4061
4062         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4063         eval { qemu_deviceadd($vmid, $devicefull); };
4064         if (my $err = $@) {
4065             eval { qemu_drivedel($vmid, $deviceid); };
4066             warn $@ if $@;
4067             die $err;
4068         }
4069
4070     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4071
4072         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4073
4074         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4075         my $use_old_bios_files = undef;
4076         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4077
4078         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4079         qemu_deviceadd($vmid, $netdevicefull);
4080         eval { qemu_deviceaddverify($vmid, $deviceid); };
4081         if (my $err = $@) {
4082             eval { qemu_netdevdel($vmid, $deviceid); };
4083             warn $@ if $@;
4084             die $err;
4085         }
4086
4087     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4088
4089         my $bridgeid = $2;
4090         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4091         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4092
4093         qemu_deviceadd($vmid, $devicefull);
4094         qemu_deviceaddverify($vmid, $deviceid);
4095
4096     } else {
4097         die "can't hotplug device '$deviceid'\n";
4098     }
4099
4100     return 1;
4101 }
4102
4103 # fixme: this should raise exceptions on error!
4104 sub vm_deviceunplug {
4105     my ($vmid, $conf, $deviceid) = @_;
4106
4107     my $devices_list = vm_devices_list($vmid);
4108     return 1 if !defined($devices_list->{$deviceid});
4109
4110     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4111
4112     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4113
4114         qemu_devicedel($vmid, $deviceid);
4115
4116     } elsif ($deviceid =~ m/^usb\d+$/) {
4117
4118         die "usb hotplug currently not reliable\n";
4119         # when unplugging usb devices this way,
4120         # there may be remaining usb controllers/hubs
4121         # so we disable it for now
4122         qemu_devicedel($vmid, $deviceid);
4123         qemu_devicedelverify($vmid, $deviceid);
4124
4125     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4126
4127         qemu_devicedel($vmid, $deviceid);
4128         qemu_devicedelverify($vmid, $deviceid);
4129         qemu_drivedel($vmid, $deviceid);
4130         qemu_iothread_del($conf, $vmid, $deviceid);
4131
4132     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4133
4134         qemu_devicedel($vmid, $deviceid);
4135         qemu_devicedelverify($vmid, $deviceid);
4136         qemu_iothread_del($conf, $vmid, $deviceid);
4137
4138     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4139
4140         qemu_devicedel($vmid, $deviceid);
4141         qemu_drivedel($vmid, $deviceid);
4142         qemu_deletescsihw($conf, $vmid, $deviceid);
4143
4144     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4145
4146         qemu_devicedel($vmid, $deviceid);
4147         qemu_devicedelverify($vmid, $deviceid);
4148         qemu_netdevdel($vmid, $deviceid);
4149
4150     } else {
4151         die "can't unplug device '$deviceid'\n";
4152     }
4153
4154     return 1;
4155 }
4156
4157 sub qemu_deviceadd {
4158     my ($vmid, $devicefull) = @_;
4159
4160     $devicefull = "driver=".$devicefull;
4161     my %options =  split(/[=,]/, $devicefull);
4162
4163     vm_mon_cmd($vmid, "device_add" , %options);
4164 }
4165
4166 sub qemu_devicedel {
4167     my ($vmid, $deviceid) = @_;
4168
4169     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4170 }
4171
4172 sub qemu_iothread_add {
4173     my($vmid, $deviceid, $device) = @_;
4174
4175     if ($device->{iothread}) {
4176         my $iothreads = vm_iothreads_list($vmid);
4177         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4178     }
4179 }
4180
4181 sub qemu_iothread_del {
4182     my($conf, $vmid, $deviceid) = @_;
4183
4184     my $device = parse_drive($deviceid, $conf->{$deviceid});
4185     if ($device->{iothread}) {
4186         my $iothreads = vm_iothreads_list($vmid);
4187         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4188     }
4189 }
4190
4191 sub qemu_objectadd {
4192     my($vmid, $objectid, $qomtype) = @_;
4193
4194     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4195
4196     return 1;
4197 }
4198
4199 sub qemu_objectdel {
4200     my($vmid, $objectid) = @_;
4201
4202     vm_mon_cmd($vmid, "object-del", id => $objectid);
4203
4204     return 1;
4205 }
4206
4207 sub qemu_driveadd {
4208     my ($storecfg, $vmid, $device) = @_;
4209
4210     my $drive = print_drive_full($storecfg, $vmid, $device);
4211     $drive =~ s/\\/\\\\/g;
4212     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4213
4214     # If the command succeeds qemu prints: "OK"
4215     return 1 if $ret =~ m/OK/s;
4216
4217     die "adding drive failed: $ret\n";
4218 }
4219
4220 sub qemu_drivedel {
4221     my($vmid, $deviceid) = @_;
4222
4223     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4224     $ret =~ s/^\s+//;
4225
4226     return 1 if $ret eq "";
4227
4228     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4229     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4230
4231     die "deleting drive $deviceid failed : $ret\n";
4232 }
4233
4234 sub qemu_deviceaddverify {
4235     my ($vmid, $deviceid) = @_;
4236
4237     for (my $i = 0; $i <= 5; $i++) {
4238          my $devices_list = vm_devices_list($vmid);
4239          return 1 if defined($devices_list->{$deviceid});
4240          sleep 1;
4241     }
4242
4243     die "error on hotplug device '$deviceid'\n";
4244 }
4245
4246
4247 sub qemu_devicedelverify {
4248     my ($vmid, $deviceid) = @_;
4249
4250     # need to verify that the device is correctly removed as device_del
4251     # is async and empty return is not reliable
4252
4253     for (my $i = 0; $i <= 5; $i++) {
4254          my $devices_list = vm_devices_list($vmid);
4255          return 1 if !defined($devices_list->{$deviceid});
4256          sleep 1;
4257     }
4258
4259     die "error on hot-unplugging device '$deviceid'\n";
4260 }
4261
4262 sub qemu_findorcreatescsihw {
4263     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4264
4265     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4266
4267     my $scsihwid="$controller_prefix$controller";
4268     my $devices_list = vm_devices_list($vmid);
4269
4270     if(!defined($devices_list->{$scsihwid})) {
4271         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4272     }
4273
4274     return 1;
4275 }
4276
4277 sub qemu_deletescsihw {
4278     my ($conf, $vmid, $opt) = @_;
4279
4280     my $device = parse_drive($opt, $conf->{$opt});
4281
4282     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4283         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4284         return 1;
4285     }
4286
4287     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4288
4289     my $devices_list = vm_devices_list($vmid);
4290     foreach my $opt (keys %{$devices_list}) {
4291         if (PVE::QemuServer::is_valid_drivename($opt)) {
4292             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4293             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4294                 return 1;
4295             }
4296         }
4297     }
4298
4299     my $scsihwid="scsihw$controller";
4300
4301     vm_deviceunplug($vmid, $conf, $scsihwid);
4302
4303     return 1;
4304 }
4305
4306 sub qemu_add_pci_bridge {
4307     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4308
4309     my $bridges = {};
4310
4311     my $bridgeid;
4312
4313     print_pci_addr($device, $bridges, $arch, $machine_type);
4314
4315     while (my ($k, $v) = each %$bridges) {
4316         $bridgeid = $k;
4317     }
4318     return 1 if !defined($bridgeid) || $bridgeid < 1;
4319
4320     my $bridge = "pci.$bridgeid";
4321     my $devices_list = vm_devices_list($vmid);
4322
4323     if (!defined($devices_list->{$bridge})) {
4324         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4325     }
4326
4327     return 1;
4328 }
4329
4330 sub qemu_set_link_status {
4331     my ($vmid, $device, $up) = @_;
4332
4333     vm_mon_cmd($vmid, "set_link", name => $device,
4334                up => $up ? JSON::true : JSON::false);
4335 }
4336
4337 sub qemu_netdevadd {
4338     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4339
4340     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4341     my %options =  split(/[=,]/, $netdev);
4342
4343     vm_mon_cmd($vmid, "netdev_add",  %options);
4344     return 1;
4345 }
4346
4347 sub qemu_netdevdel {
4348     my ($vmid, $deviceid) = @_;
4349
4350     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4351 }
4352
4353 sub qemu_usb_hotplug {
4354     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4355
4356     return if !$device;
4357
4358     # remove the old one first
4359     vm_deviceunplug($vmid, $conf, $deviceid);
4360
4361     # check if xhci controller is necessary and available
4362     if ($device->{usb3}) {
4363
4364         my $devicelist = vm_devices_list($vmid);
4365
4366         if (!$devicelist->{xhci}) {
4367             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4368             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4369         }
4370     }
4371     my $d = parse_usb_device($device->{host});
4372     $d->{usb3} = $device->{usb3};
4373
4374     # add the new one
4375     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4376 }
4377
4378 sub qemu_cpu_hotplug {
4379     my ($vmid, $conf, $vcpus) = @_;
4380
4381     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4382
4383     my $sockets = 1;
4384     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4385     $sockets = $conf->{sockets} if  $conf->{sockets};
4386     my $cores = $conf->{cores} || 1;
4387     my $maxcpus = $sockets * $cores;
4388
4389     $vcpus = $maxcpus if !$vcpus;
4390
4391     die "you can't add more vcpus than maxcpus\n"
4392         if $vcpus > $maxcpus;
4393
4394     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4395
4396     if ($vcpus < $currentvcpus) {
4397
4398         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4399
4400             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4401                 qemu_devicedel($vmid, "cpu$i");
4402                 my $retry = 0;
4403                 my $currentrunningvcpus = undef;
4404                 while (1) {
4405                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4406                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4407                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4408                     $retry++;
4409                     sleep 1;
4410                 }
4411                 #update conf after each succesfull cpu unplug
4412                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4413                 PVE::QemuConfig->write_config($vmid, $conf);
4414             }
4415         } else {
4416             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4417         }
4418
4419         return;
4420     }
4421
4422     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4423     die "vcpus in running vm does not match its configuration\n"
4424         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4425
4426     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4427
4428         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4429             my $cpustr = print_cpu_device($conf, $i);
4430             qemu_deviceadd($vmid, $cpustr);
4431
4432             my $retry = 0;
4433             my $currentrunningvcpus = undef;
4434             while (1) {
4435                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4436                 last if scalar(@{$currentrunningvcpus}) == $i;
4437                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4438                 sleep 1;
4439                 $retry++;
4440             }
4441             #update conf after each succesfull cpu hotplug
4442             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4443             PVE::QemuConfig->write_config($vmid, $conf);
4444         }
4445     } else {
4446
4447         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4448             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4449         }
4450     }
4451 }
4452
4453 sub qemu_block_set_io_throttle {
4454     my ($vmid, $deviceid,
4455         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4456         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4457         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4458         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4459
4460     return if !check_running($vmid) ;
4461
4462     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4463         bps => int($bps),
4464         bps_rd => int($bps_rd),
4465         bps_wr => int($bps_wr),
4466         iops => int($iops),
4467         iops_rd => int($iops_rd),
4468         iops_wr => int($iops_wr),
4469         bps_max => int($bps_max),
4470         bps_rd_max => int($bps_rd_max),
4471         bps_wr_max => int($bps_wr_max),
4472         iops_max => int($iops_max),
4473         iops_rd_max => int($iops_rd_max),
4474         iops_wr_max => int($iops_wr_max),
4475         bps_max_length => int($bps_max_length),
4476         bps_rd_max_length => int($bps_rd_max_length),
4477         bps_wr_max_length => int($bps_wr_max_length),
4478         iops_max_length => int($iops_max_length),
4479         iops_rd_max_length => int($iops_rd_max_length),
4480         iops_wr_max_length => int($iops_wr_max_length),
4481     );
4482
4483 }
4484
4485 # old code, only used to shutdown old VM after update
4486 sub __read_avail {
4487     my ($fh, $timeout) = @_;
4488
4489     my $sel = new IO::Select;
4490     $sel->add($fh);
4491
4492     my $res = '';
4493     my $buf;
4494
4495     my @ready;
4496     while (scalar (@ready = $sel->can_read($timeout))) {
4497         my $count;
4498         if ($count = $fh->sysread($buf, 8192)) {
4499             if ($buf =~ /^(.*)\(qemu\) $/s) {
4500                 $res .= $1;
4501                 last;
4502             } else {
4503                 $res .= $buf;
4504             }
4505         } else {
4506             if (!defined($count)) {
4507                 die "$!\n";
4508             }
4509             last;
4510         }
4511     }
4512
4513     die "monitor read timeout\n" if !scalar(@ready);
4514
4515     return $res;
4516 }
4517
4518 sub qemu_block_resize {
4519     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4520
4521     my $running = check_running($vmid);
4522
4523     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4524
4525     return if !$running;
4526
4527     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4528
4529 }
4530
4531 sub qemu_volume_snapshot {
4532     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4533
4534     my $running = check_running($vmid);
4535
4536     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4537         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4538     } else {
4539         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4540     }
4541 }
4542
4543 sub qemu_volume_snapshot_delete {
4544     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4545
4546     my $running = check_running($vmid);
4547
4548     if($running) {
4549
4550         $running = undef;
4551         my $conf = PVE::QemuConfig->load_config($vmid);
4552         foreach_drive($conf, sub {
4553             my ($ds, $drive) = @_;
4554             $running = 1 if $drive->{file} eq $volid;
4555         });
4556     }
4557
4558     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4559         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4560     } else {
4561         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4562     }
4563 }
4564
4565 sub set_migration_caps {
4566     my ($vmid) = @_;
4567
4568     my $cap_ref = [];
4569
4570     my $enabled_cap = {
4571         "auto-converge" => 1,
4572         "xbzrle" => 1,
4573         "x-rdma-pin-all" => 0,
4574         "zero-blocks" => 0,
4575         "compress" => 0
4576     };
4577
4578     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4579
4580     for my $supported_capability (@$supported_capabilities) {
4581         push @$cap_ref, {
4582             capability => $supported_capability->{capability},
4583             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4584         };
4585     }
4586
4587     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4588 }
4589
4590 my $fast_plug_option = {
4591     'lock' => 1,
4592     'name' => 1,
4593     'onboot' => 1,
4594     'shares' => 1,
4595     'startup' => 1,
4596     'description' => 1,
4597     'protection' => 1,
4598     'vmstatestorage' => 1,
4599 };
4600
4601 # hotplug changes in [PENDING]
4602 # $selection hash can be used to only apply specified options, for
4603 # example: { cores => 1 } (only apply changed 'cores')
4604 # $errors ref is used to return error messages
4605 sub vmconfig_hotplug_pending {
4606     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4607
4608     my $defaults = load_defaults();
4609     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4610
4611     # commit values which do not have any impact on running VM first
4612     # Note: those option cannot raise errors, we we do not care about
4613     # $selection and always apply them.
4614
4615     my $add_error = sub {
4616         my ($opt, $msg) = @_;
4617         $errors->{$opt} = "hotplug problem - $msg";
4618     };
4619
4620     my $changes = 0;
4621     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4622         if ($fast_plug_option->{$opt}) {
4623             $conf->{$opt} = $conf->{pending}->{$opt};
4624             delete $conf->{pending}->{$opt};
4625             $changes = 1;
4626         }
4627     }
4628
4629     if ($changes) {
4630         PVE::QemuConfig->write_config($vmid, $conf);
4631         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4632     }
4633
4634     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4635
4636     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4637     while (my ($opt, $force) = each %$pending_delete_hash) {
4638         next if $selection && !$selection->{$opt};
4639         eval {
4640             if ($opt eq 'hotplug') {
4641                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4642             } elsif ($opt eq 'tablet') {
4643                 die "skip\n" if !$hotplug_features->{usb};
4644                 if ($defaults->{tablet}) {
4645                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4646                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4647                         if $arch eq 'aarch64';
4648                 } else {
4649                     vm_deviceunplug($vmid, $conf, 'tablet');
4650                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4651                 }
4652             } elsif ($opt =~ m/^usb\d+/) {
4653                 die "skip\n";
4654                 # since we cannot reliably hot unplug usb devices
4655                 # we are disabling it
4656                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4657                 vm_deviceunplug($vmid, $conf, $opt);
4658             } elsif ($opt eq 'vcpus') {
4659                 die "skip\n" if !$hotplug_features->{cpu};
4660                 qemu_cpu_hotplug($vmid, $conf, undef);
4661             } elsif ($opt eq 'balloon') {
4662                 # enable balloon device is not hotpluggable
4663                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4664                 # here we reset the ballooning value to memory
4665                 my $balloon = $conf->{memory} || $defaults->{memory};
4666                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4667             } elsif ($fast_plug_option->{$opt}) {
4668                 # do nothing
4669             } elsif ($opt =~ m/^net(\d+)$/) {
4670                 die "skip\n" if !$hotplug_features->{network};
4671                 vm_deviceunplug($vmid, $conf, $opt);
4672             } elsif (is_valid_drivename($opt)) {
4673                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4674                 vm_deviceunplug($vmid, $conf, $opt);
4675                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4676             } elsif ($opt =~ m/^memory$/) {
4677                 die "skip\n" if !$hotplug_features->{memory};
4678                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4679             } elsif ($opt eq 'cpuunits') {
4680                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4681             } elsif ($opt eq 'cpulimit') {
4682                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4683             } else {
4684                 die "skip\n";
4685             }
4686         };
4687         if (my $err = $@) {
4688             &$add_error($opt, $err) if $err ne "skip\n";
4689         } else {
4690             # save new config if hotplug was successful
4691             delete $conf->{$opt};
4692             vmconfig_undelete_pending_option($conf, $opt);
4693             PVE::QemuConfig->write_config($vmid, $conf);
4694             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4695         }
4696     }
4697
4698     my $apply_pending_cloudinit;
4699     $apply_pending_cloudinit = sub {
4700         my ($key, $value) = @_;
4701         $apply_pending_cloudinit = sub {}; # once is enough
4702
4703         my @cloudinit_opts = keys %$confdesc_cloudinit;
4704         foreach my $opt (keys %{$conf->{pending}}) {
4705             next if !grep { $_ eq $opt } @cloudinit_opts;
4706             $conf->{$opt} = delete $conf->{pending}->{$opt};
4707         }
4708
4709         my $new_conf = { %$conf };
4710         $new_conf->{$key} = $value;
4711         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4712     };
4713
4714     foreach my $opt (keys %{$conf->{pending}}) {
4715         next if $selection && !$selection->{$opt};
4716         my $value = $conf->{pending}->{$opt};
4717         eval {
4718             if ($opt eq 'hotplug') {
4719                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4720             } elsif ($opt eq 'tablet') {
4721                 die "skip\n" if !$hotplug_features->{usb};
4722                 if ($value == 1) {
4723                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4724                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4725                         if $arch eq 'aarch64';
4726                 } elsif ($value == 0) {
4727                     vm_deviceunplug($vmid, $conf, 'tablet');
4728                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4729                 }
4730             } elsif ($opt =~ m/^usb\d+$/) {
4731                 die "skip\n";
4732                 # since we cannot reliably hot unplug usb devices
4733                 # we are disabling it
4734                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4735                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4736                 die "skip\n" if !$d;
4737                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4738             } elsif ($opt eq 'vcpus') {
4739                 die "skip\n" if !$hotplug_features->{cpu};
4740                 qemu_cpu_hotplug($vmid, $conf, $value);
4741             } elsif ($opt eq 'balloon') {
4742                 # enable/disable balloning device is not hotpluggable
4743                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4744                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4745                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4746
4747                 # allow manual ballooning if shares is set to zero
4748                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4749                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4750                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4751                 }
4752             } elsif ($opt =~ m/^net(\d+)$/) {
4753                 # some changes can be done without hotplug
4754                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4755                                     $vmid, $opt, $value, $arch, $machine_type);
4756             } elsif (is_valid_drivename($opt)) {
4757                 # some changes can be done without hotplug
4758                 my $drive = parse_drive($opt, $value);
4759                 if (drive_is_cloudinit($drive)) {
4760                     &$apply_pending_cloudinit($opt, $value);
4761                 }
4762                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4763                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4764             } elsif ($opt =~ m/^memory$/) { #dimms
4765                 die "skip\n" if !$hotplug_features->{memory};
4766                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4767             } elsif ($opt eq 'cpuunits') {
4768                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4769             } elsif ($opt eq 'cpulimit') {
4770                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4771                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4772             } else {
4773                 die "skip\n";  # skip non-hot-pluggable options
4774             }
4775         };
4776         if (my $err = $@) {
4777             &$add_error($opt, $err) if $err ne "skip\n";
4778         } else {
4779             # save new config if hotplug was successful
4780             $conf->{$opt} = $value;
4781             delete $conf->{pending}->{$opt};
4782             PVE::QemuConfig->write_config($vmid, $conf);
4783             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4784         }
4785     }
4786 }
4787
4788 sub try_deallocate_drive {
4789     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4790
4791     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4792         my $volid = $drive->{file};
4793         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4794             my $sid = PVE::Storage::parse_volume_id($volid);
4795             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4796
4797             # check if the disk is really unused
4798             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4799                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4800             PVE::Storage::vdisk_free($storecfg, $volid);
4801             return 1;
4802         } else {
4803             # If vm is not owner of this disk remove from config
4804             return 1;
4805         }
4806     }
4807
4808     return undef;
4809 }
4810
4811 sub vmconfig_delete_or_detach_drive {
4812     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4813
4814     my $drive = parse_drive($opt, $conf->{$opt});
4815
4816     my $rpcenv = PVE::RPCEnvironment::get();
4817     my $authuser = $rpcenv->get_user();
4818
4819     if ($force) {
4820         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4821         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4822     } else {
4823         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4824     }
4825 }
4826
4827 sub vmconfig_apply_pending {
4828     my ($vmid, $conf, $storecfg) = @_;
4829
4830     # cold plug
4831
4832     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4833     while (my ($opt, $force) = each %$pending_delete_hash) {
4834         die "internal error" if $opt =~ m/^unused/;
4835         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4836         if (!defined($conf->{$opt})) {
4837             vmconfig_undelete_pending_option($conf, $opt);
4838             PVE::QemuConfig->write_config($vmid, $conf);
4839         } elsif (is_valid_drivename($opt)) {
4840             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4841             vmconfig_undelete_pending_option($conf, $opt);
4842             delete $conf->{$opt};
4843             PVE::QemuConfig->write_config($vmid, $conf);
4844         } else {
4845             vmconfig_undelete_pending_option($conf, $opt);
4846             delete $conf->{$opt};
4847             PVE::QemuConfig->write_config($vmid, $conf);
4848         }
4849     }
4850
4851     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4852
4853     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4854         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4855
4856         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4857             # skip if nothing changed
4858         } elsif (is_valid_drivename($opt)) {
4859             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4860                 if defined($conf->{$opt});
4861             $conf->{$opt} = $conf->{pending}->{$opt};
4862         } else {
4863             $conf->{$opt} = $conf->{pending}->{$opt};
4864         }
4865
4866         delete $conf->{pending}->{$opt};
4867         PVE::QemuConfig->write_config($vmid, $conf);
4868     }
4869 }
4870
4871 my $safe_num_ne = sub {
4872     my ($a, $b) = @_;
4873
4874     return 0 if !defined($a) && !defined($b);
4875     return 1 if !defined($a);
4876     return 1 if !defined($b);
4877
4878     return $a != $b;
4879 };
4880
4881 my $safe_string_ne = sub {
4882     my ($a, $b) = @_;
4883
4884     return 0 if !defined($a) && !defined($b);
4885     return 1 if !defined($a);
4886     return 1 if !defined($b);
4887
4888     return $a ne $b;
4889 };
4890
4891 sub vmconfig_update_net {
4892     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4893
4894     my $newnet = parse_net($value);
4895
4896     if ($conf->{$opt}) {
4897         my $oldnet = parse_net($conf->{$opt});
4898
4899         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4900             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4901             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4902             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4903
4904             # for non online change, we try to hot-unplug
4905             die "skip\n" if !$hotplug;
4906             vm_deviceunplug($vmid, $conf, $opt);
4907         } else {
4908
4909             die "internal error" if $opt !~ m/net(\d+)/;
4910             my $iface = "tap${vmid}i$1";
4911
4912             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4913                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4914                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4915                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4916                 PVE::Network::tap_unplug($iface);
4917                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4918             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4919                 # Rate can be applied on its own but any change above needs to
4920                 # include the rate in tap_plug since OVS resets everything.
4921                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4922             }
4923
4924             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4925                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4926             }
4927
4928             return 1;
4929         }
4930     }
4931
4932     if ($hotplug) {
4933         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4934     } else {
4935         die "skip\n";
4936     }
4937 }
4938
4939 sub vmconfig_update_disk {
4940     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4941
4942     # fixme: do we need force?
4943
4944     my $drive = parse_drive($opt, $value);
4945
4946     if ($conf->{$opt}) {
4947
4948         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4949
4950             my $media = $drive->{media} || 'disk';
4951             my $oldmedia = $old_drive->{media} || 'disk';
4952             die "unable to change media type\n" if $media ne $oldmedia;
4953
4954             if (!drive_is_cdrom($old_drive)) {
4955
4956                 if ($drive->{file} ne $old_drive->{file}) {
4957
4958                     die "skip\n" if !$hotplug;
4959
4960                     # unplug and register as unused
4961                     vm_deviceunplug($vmid, $conf, $opt);
4962                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4963
4964                 } else {
4965                     # update existing disk
4966
4967                     # skip non hotpluggable value
4968                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4969                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4970                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4971                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4972                         die "skip\n";
4973                     }
4974
4975                     # apply throttle
4976                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4977                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4978                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4979                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4980                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4981                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4982                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4983                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4984                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4985                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4986                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4987                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4988                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4989                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4990                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4991                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4992                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4993                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4994
4995                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4996                                                    ($drive->{mbps} || 0)*1024*1024,
4997                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4998                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4999                                                    $drive->{iops} || 0,
5000                                                    $drive->{iops_rd} || 0,
5001                                                    $drive->{iops_wr} || 0,
5002                                                    ($drive->{mbps_max} || 0)*1024*1024,
5003                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5004                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5005                                                    $drive->{iops_max} || 0,
5006                                                    $drive->{iops_rd_max} || 0,
5007                                                    $drive->{iops_wr_max} || 0,
5008                                                    $drive->{bps_max_length} || 1,
5009                                                    $drive->{bps_rd_max_length} || 1,
5010                                                    $drive->{bps_wr_max_length} || 1,
5011                                                    $drive->{iops_max_length} || 1,
5012                                                    $drive->{iops_rd_max_length} || 1,
5013                                                    $drive->{iops_wr_max_length} || 1);
5014
5015                     }
5016
5017                     return 1;
5018                 }
5019
5020             } else { # cdrom
5021
5022                 if ($drive->{file} eq 'none') {
5023                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5024                     if (drive_is_cloudinit($old_drive)) {
5025                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5026                     }
5027                 } else {
5028                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5029                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5030                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5031                 }
5032
5033                 return 1;
5034             }
5035         }
5036     }
5037
5038     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5039     # hotplug new disks
5040     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5041     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5042 }
5043
5044 sub vm_start {
5045     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5046         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5047
5048     PVE::QemuConfig->lock_config($vmid, sub {
5049         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5050
5051         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5052
5053         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5054
5055         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5056
5057         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5058             vmconfig_apply_pending($vmid, $conf, $storecfg);
5059             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5060         }
5061
5062         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5063
5064         my $defaults = load_defaults();
5065
5066         # set environment variable useful inside network script
5067         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5068
5069         my $local_volumes = {};
5070
5071         if ($targetstorage) {
5072             foreach_drive($conf, sub {
5073                 my ($ds, $drive) = @_;
5074
5075                 return if drive_is_cdrom($drive);
5076
5077                 my $volid = $drive->{file};
5078
5079                 return if !$volid;
5080
5081                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5082
5083                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5084                 return if $scfg->{shared};
5085                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5086             });
5087
5088             my $format = undef;
5089
5090             foreach my $opt (sort keys %$local_volumes) {
5091
5092                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5093                 my $drive = parse_drive($opt, $conf->{$opt});
5094
5095                 #if remote storage is specified, use default format
5096                 if ($targetstorage && $targetstorage ne "1") {
5097                     $storeid = $targetstorage;
5098                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5099                     $format = $defFormat;
5100                 } else {
5101                     #else we use same format than original
5102                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5103                     $format = qemu_img_format($scfg, $volid);
5104                 }
5105
5106                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5107                 my $newdrive = $drive;
5108                 $newdrive->{format} = $format;
5109                 $newdrive->{file} = $newvolid;
5110                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5111                 $local_volumes->{$opt} = $drivestr;
5112                 #pass drive to conf for command line
5113                 $conf->{$opt} = $drivestr;
5114             }
5115         }
5116
5117         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5118
5119         my $migrate_port = 0;
5120         my $migrate_uri;
5121         if ($statefile) {
5122             if ($statefile eq 'tcp') {
5123                 my $localip = "localhost";
5124                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5125                 my $nodename = PVE::INotify::nodename();
5126
5127                 if (!defined($migration_type)) {
5128                     if (defined($datacenterconf->{migration}->{type})) {
5129                         $migration_type = $datacenterconf->{migration}->{type};
5130                     } else {
5131                         $migration_type = 'secure';
5132                     }
5133                 }
5134
5135                 if ($migration_type eq 'insecure') {
5136                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5137                     if ($migrate_network_addr) {
5138                         $localip = $migrate_network_addr;
5139                     } else {
5140                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5141                     }
5142
5143                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5144                 }
5145
5146                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5147                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5148                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5149                 push @$cmd, '-incoming', $migrate_uri;
5150                 push @$cmd, '-S';
5151
5152             } elsif ($statefile eq 'unix') {
5153                 # should be default for secure migrations as a ssh TCP forward
5154                 # tunnel is not deterministic reliable ready and fails regurarly
5155                 # to set up in time, so use UNIX socket forwards
5156                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5157                 unlink $socket_addr;
5158
5159                 $migrate_uri = "unix:$socket_addr";
5160
5161                 push @$cmd, '-incoming', $migrate_uri;
5162                 push @$cmd, '-S';
5163
5164             } else {
5165                 push @$cmd, '-loadstate', $statefile;
5166             }
5167         } elsif ($paused) {
5168             push @$cmd, '-S';
5169         }
5170
5171         # host pci devices
5172         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5173           my $d = parse_hostpci($conf->{"hostpci$i"});
5174           next if !$d;
5175           my $pcidevices = $d->{pciid};
5176           foreach my $pcidevice (@$pcidevices) {
5177                 my $pciid = $pcidevice->{id};
5178
5179                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5180                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5181                 die "no pci device info for device '$pciid'\n" if !$info;
5182
5183                 if ($d->{mdev}) {
5184                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5185                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5186                 } else {
5187                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5188                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5189                     die "can't reset pci device '$pciid'\n"
5190                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5191                 }
5192           }
5193         }
5194
5195         PVE::Storage::activate_volumes($storecfg, $vollist);
5196
5197         if (!check_running($vmid, 1)) {
5198             eval {
5199                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5200                     outfunc => sub {}, errfunc => sub {});
5201             };
5202         }
5203
5204         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5205                                                   : $defaults->{cpuunits};
5206
5207         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5208         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5209
5210         my %properties = (
5211             Slice => 'qemu.slice',
5212             KillMode => 'none',
5213             CPUShares => $cpuunits
5214         );
5215
5216         if (my $cpulimit = $conf->{cpulimit}) {
5217             $properties{CPUQuota} = int($cpulimit * 100);
5218         }
5219         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5220
5221         my $run_qemu = sub {
5222             PVE::Tools::run_fork sub {
5223                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5224                 run_command($cmd, %run_params);
5225             };
5226         };
5227
5228         if ($conf->{hugepages}) {
5229
5230             my $code = sub {
5231                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5232                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5233
5234                 PVE::QemuServer::Memory::hugepages_mount();
5235                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5236
5237                 eval { $run_qemu->() };
5238                 if (my $err = $@) {
5239                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5240                     die $err;
5241                 }
5242
5243                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5244             };
5245             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5246
5247         } else {
5248             eval { $run_qemu->() };
5249         }
5250
5251         if (my $err = $@) {
5252             # deactivate volumes if start fails
5253             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5254             die "start failed: $err";
5255         }
5256
5257         print "migration listens on $migrate_uri\n" if $migrate_uri;
5258
5259         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5260             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5261             warn $@ if $@;
5262         }
5263
5264         #start nbd server for storage migration
5265         if ($targetstorage) {
5266             my $nodename = PVE::INotify::nodename();
5267             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5268             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5269             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5270             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5271
5272             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5273
5274             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5275
5276             foreach my $opt (sort keys %$local_volumes) {
5277                 my $volid = $local_volumes->{$opt};
5278                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5279                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5280                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5281             }
5282         }
5283
5284         if ($migratedfrom) {
5285             eval {
5286                 set_migration_caps($vmid);
5287             };
5288             warn $@ if $@;
5289
5290             if ($spice_port) {
5291                 print "spice listens on port $spice_port\n";
5292                 if ($spice_ticket) {
5293                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5294                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5295                 }
5296             }
5297
5298         } else {
5299             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5300                 if !$statefile && $conf->{balloon};
5301
5302             foreach my $opt (keys %$conf) {
5303                 next if $opt !~  m/^net\d+$/;
5304                 my $nicconf = parse_net($conf->{$opt});
5305                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5306             }
5307         }
5308
5309         vm_mon_cmd_nocheck($vmid, 'qom-set',
5310                     path => "machine/peripheral/balloon0",
5311                     property => "guest-stats-polling-interval",
5312                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5313
5314     });
5315 }
5316
5317 sub vm_mon_cmd {
5318     my ($vmid, $execute, %params) = @_;
5319
5320     my $cmd = { execute => $execute, arguments => \%params };
5321     vm_qmp_command($vmid, $cmd);
5322 }
5323
5324 sub vm_mon_cmd_nocheck {
5325     my ($vmid, $execute, %params) = @_;
5326
5327     my $cmd = { execute => $execute, arguments => \%params };
5328     vm_qmp_command($vmid, $cmd, 1);
5329 }
5330
5331 sub vm_qmp_command {
5332     my ($vmid, $cmd, $nocheck) = @_;
5333
5334     my $res;
5335
5336     my $timeout;
5337     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5338         $timeout = $cmd->{arguments}->{timeout};
5339         delete $cmd->{arguments}->{timeout};
5340     }
5341
5342     eval {
5343         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5344         my $sname = qmp_socket($vmid);
5345         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5346             my $qmpclient = PVE::QMPClient->new();
5347
5348             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5349         } else {
5350             die "unable to open monitor socket\n";
5351         }
5352     };
5353     if (my $err = $@) {
5354         syslog("err", "VM $vmid qmp command failed - $err");
5355         die $err;
5356     }
5357
5358     return $res;
5359 }
5360
5361 sub vm_human_monitor_command {
5362     my ($vmid, $cmdline) = @_;
5363
5364     my $res;
5365
5366     my $cmd = {
5367         execute => 'human-monitor-command',
5368         arguments => { 'command-line' => $cmdline},
5369     };
5370
5371     return vm_qmp_command($vmid, $cmd);
5372 }
5373
5374 sub vm_commandline {
5375     my ($storecfg, $vmid, $snapname) = @_;
5376
5377     my $conf = PVE::QemuConfig->load_config($vmid);
5378
5379     if ($snapname) {
5380         my $snapshot = $conf->{snapshots}->{$snapname};
5381         die "snapshot '$snapname' does not exist\n"
5382             if !defined($snapshot);
5383         my $digest = $conf->{digest};
5384
5385         # we need the digest of the file
5386         $snapshot->{digest} = $conf->{digest};
5387         $conf = $snapshot;
5388     }
5389
5390     my $defaults = load_defaults();
5391
5392     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5393
5394     return PVE::Tools::cmd2string($cmd);
5395 }
5396
5397 sub vm_reset {
5398     my ($vmid, $skiplock) = @_;
5399
5400     PVE::QemuConfig->lock_config($vmid, sub {
5401
5402         my $conf = PVE::QemuConfig->load_config($vmid);
5403
5404         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5405
5406         vm_mon_cmd($vmid, "system_reset");
5407     });
5408 }
5409
5410 sub get_vm_volumes {
5411     my ($conf) = @_;
5412
5413     my $vollist = [];
5414     foreach_volid($conf, sub {
5415         my ($volid, $attr) = @_;
5416
5417         return if $volid =~ m|^/|;
5418
5419         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5420         return if !$sid;
5421
5422         push @$vollist, $volid;
5423     });
5424
5425     return $vollist;
5426 }
5427
5428 sub vm_stop_cleanup {
5429     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5430
5431     eval {
5432
5433         if (!$keepActive) {
5434             my $vollist = get_vm_volumes($conf);
5435             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5436         }
5437
5438         foreach my $ext (qw(mon qmp pid vnc qga)) {
5439             unlink "/var/run/qemu-server/${vmid}.$ext";
5440         }
5441
5442         foreach my $key (keys %$conf) {
5443             next if $key !~ m/^hostpci(\d+)$/;
5444             my $hostpciindex = $1;
5445             my $d = parse_hostpci($conf->{$key});
5446             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5447
5448             foreach my $pci (@{$d->{pciid}}) {
5449                 my $pciid = $pci->{id};
5450                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5451             }
5452         }
5453
5454         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5455     };
5456     warn $@ if $@; # avoid errors - just warn
5457 }
5458
5459 # Note: use $nockeck to skip tests if VM configuration file exists.
5460 # We need that when migration VMs to other nodes (files already moved)
5461 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5462 sub vm_stop {
5463     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5464
5465     $force = 1 if !defined($force) && !$shutdown;
5466
5467     if ($migratedfrom){
5468         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5469         kill 15, $pid if $pid;
5470         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5471         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5472         return;
5473     }
5474
5475     PVE::QemuConfig->lock_config($vmid, sub {
5476
5477         my $pid = check_running($vmid, $nocheck);
5478         return if !$pid;
5479
5480         my $conf;
5481         if (!$nocheck) {
5482             $conf = PVE::QemuConfig->load_config($vmid);
5483             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5484             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5485                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5486                 $timeout = $opts->{down} if $opts->{down};
5487             }
5488         }
5489
5490         $timeout = 60 if !defined($timeout);
5491
5492         eval {
5493             if ($shutdown) {
5494                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5495                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5496                 } else {
5497                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5498                 }
5499             } else {
5500                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5501             }
5502         };
5503         my $err = $@;
5504
5505         if (!$err) {
5506             my $count = 0;
5507             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5508                 $count++;
5509                 sleep 1;
5510             }
5511
5512             if ($count >= $timeout) {
5513                 if ($force) {
5514                     warn "VM still running - terminating now with SIGTERM\n";
5515                     kill 15, $pid;
5516                 } else {
5517                     die "VM quit/powerdown failed - got timeout\n";
5518                 }
5519             } else {
5520                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5521                 return;
5522             }
5523         } else {
5524             if ($force) {
5525                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5526                 kill 15, $pid;
5527             } else {
5528                 die "VM quit/powerdown failed\n";
5529             }
5530         }
5531
5532         # wait again
5533         $timeout = 10;
5534
5535         my $count = 0;
5536         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5537             $count++;
5538             sleep 1;
5539         }
5540
5541         if ($count >= $timeout) {
5542             warn "VM still running - terminating now with SIGKILL\n";
5543             kill 9, $pid;
5544             sleep 1;
5545         }
5546
5547         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5548    });
5549 }
5550
5551 sub vm_suspend {
5552     my ($vmid, $skiplock) = @_;
5553
5554     PVE::QemuConfig->lock_config($vmid, sub {
5555
5556         my $conf = PVE::QemuConfig->load_config($vmid);
5557
5558         PVE::QemuConfig->check_lock($conf)
5559             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5560
5561         vm_mon_cmd($vmid, "stop");
5562     });
5563 }
5564
5565 sub vm_resume {
5566     my ($vmid, $skiplock, $nocheck) = @_;
5567
5568     PVE::QemuConfig->lock_config($vmid, sub {
5569
5570         my $res = vm_mon_cmd($vmid, 'query-status');
5571         my $resume_cmd = 'cont';
5572
5573         if ($res->{status} && $res->{status} eq 'suspended') {
5574             $resume_cmd = 'system_wakeup';
5575         }
5576
5577         if (!$nocheck) {
5578
5579             my $conf = PVE::QemuConfig->load_config($vmid);
5580
5581             PVE::QemuConfig->check_lock($conf)
5582                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5583
5584             vm_mon_cmd($vmid, $resume_cmd);
5585
5586         } else {
5587             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5588         }
5589     });
5590 }
5591
5592 sub vm_sendkey {
5593     my ($vmid, $skiplock, $key) = @_;
5594
5595     PVE::QemuConfig->lock_config($vmid, sub {
5596
5597         my $conf = PVE::QemuConfig->load_config($vmid);
5598
5599         # there is no qmp command, so we use the human monitor command
5600         vm_human_monitor_command($vmid, "sendkey $key");
5601     });
5602 }
5603
5604 sub vm_destroy {
5605     my ($storecfg, $vmid, $skiplock) = @_;
5606
5607     PVE::QemuConfig->lock_config($vmid, sub {
5608
5609         my $conf = PVE::QemuConfig->load_config($vmid);
5610
5611         if (!check_running($vmid)) {
5612             destroy_vm($storecfg, $vmid, undef, $skiplock);
5613         } else {
5614             die "VM $vmid is running - destroy failed\n";
5615         }
5616     });
5617 }
5618
5619 # vzdump restore implementaion
5620
5621 sub tar_archive_read_firstfile {
5622     my $archive = shift;
5623
5624     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5625
5626     # try to detect archive type first
5627     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5628         die "unable to open file '$archive'\n";
5629     my $firstfile = <$fh>;
5630     kill 15, $pid;
5631     close $fh;
5632
5633     die "ERROR: archive contaions no data\n" if !$firstfile;
5634     chomp $firstfile;
5635
5636     return $firstfile;
5637 }
5638
5639 sub tar_restore_cleanup {
5640     my ($storecfg, $statfile) = @_;
5641
5642     print STDERR "starting cleanup\n";
5643
5644     if (my $fd = IO::File->new($statfile, "r")) {
5645         while (defined(my $line = <$fd>)) {
5646             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5647                 my $volid = $2;
5648                 eval {
5649                     if ($volid =~ m|^/|) {
5650                         unlink $volid || die 'unlink failed\n';
5651                     } else {
5652                         PVE::Storage::vdisk_free($storecfg, $volid);
5653                     }
5654                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5655                 };
5656                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5657             } else {
5658                 print STDERR "unable to parse line in statfile - $line";
5659             }
5660         }
5661         $fd->close();
5662     }
5663 }
5664
5665 sub restore_archive {
5666     my ($archive, $vmid, $user, $opts) = @_;
5667
5668     my $format = $opts->{format};
5669     my $comp;
5670
5671     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5672         $format = 'tar' if !$format;
5673         $comp = 'gzip';
5674     } elsif ($archive =~ m/\.tar$/) {
5675         $format = 'tar' if !$format;
5676     } elsif ($archive =~ m/.tar.lzo$/) {
5677         $format = 'tar' if !$format;
5678         $comp = 'lzop';
5679     } elsif ($archive =~ m/\.vma$/) {
5680         $format = 'vma' if !$format;
5681     } elsif ($archive =~ m/\.vma\.gz$/) {
5682         $format = 'vma' if !$format;
5683         $comp = 'gzip';
5684     } elsif ($archive =~ m/\.vma\.lzo$/) {
5685         $format = 'vma' if !$format;
5686         $comp = 'lzop';
5687     } else {
5688         $format = 'vma' if !$format; # default
5689     }
5690
5691     # try to detect archive format
5692     if ($format eq 'tar') {
5693         return restore_tar_archive($archive, $vmid, $user, $opts);
5694     } else {
5695         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5696     }
5697 }
5698
5699 sub restore_update_config_line {
5700     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5701
5702     return if $line =~ m/^\#qmdump\#/;
5703     return if $line =~ m/^\#vzdump\#/;
5704     return if $line =~ m/^lock:/;
5705     return if $line =~ m/^unused\d+:/;
5706     return if $line =~ m/^parent:/;
5707     return if $line =~ m/^template:/; # restored VM is never a template
5708
5709     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5710     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5711         # try to convert old 1.X settings
5712         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5713         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5714             my ($model, $macaddr) = split(/\=/, $devconfig);
5715             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5716             my $net = {
5717                 model => $model,
5718                 bridge => "vmbr$ind",
5719                 macaddr => $macaddr,
5720             };
5721             my $netstr = print_net($net);
5722
5723             print $outfd "net$cookie->{netcount}: $netstr\n";
5724             $cookie->{netcount}++;
5725         }
5726     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5727         my ($id, $netstr) = ($1, $2);
5728         my $net = parse_net($netstr);
5729         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5730         $netstr = print_net($net);
5731         print $outfd "$id: $netstr\n";
5732     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5733         my $virtdev = $1;
5734         my $value = $3;
5735         my $di = parse_drive($virtdev, $value);
5736         if (defined($di->{backup}) && !$di->{backup}) {
5737             print $outfd "#$line";
5738         } elsif ($map->{$virtdev}) {
5739             delete $di->{format}; # format can change on restore
5740             $di->{file} = $map->{$virtdev};
5741             $value = print_drive($vmid, $di);
5742             print $outfd "$virtdev: $value\n";
5743         } else {
5744             print $outfd $line;
5745         }
5746     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5747         my $vmgenid = $1;
5748         if ($vmgenid ne '0') {
5749             # always generate a new vmgenid if there was a valid one setup
5750             $vmgenid = generate_uuid();
5751         }
5752         print $outfd "vmgenid: $vmgenid\n";
5753     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5754         my ($uuid, $uuid_str);
5755         UUID::generate($uuid);
5756         UUID::unparse($uuid, $uuid_str);
5757         my $smbios1 = parse_smbios1($2);
5758         $smbios1->{uuid} = $uuid_str;
5759         print $outfd $1.print_smbios1($smbios1)."\n";
5760     } else {
5761         print $outfd $line;
5762     }
5763 }
5764
5765 sub scan_volids {
5766     my ($cfg, $vmid) = @_;
5767
5768     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5769
5770     my $volid_hash = {};
5771     foreach my $storeid (keys %$info) {
5772         foreach my $item (@{$info->{$storeid}}) {
5773             next if !($item->{volid} && $item->{size});
5774             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5775             $volid_hash->{$item->{volid}} = $item;
5776         }
5777     }
5778
5779     return $volid_hash;
5780 }
5781
5782 sub is_volume_in_use {
5783     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5784
5785     my $path = PVE::Storage::path($storecfg, $volid);
5786
5787     my $scan_config = sub {
5788         my ($cref, $snapname) = @_;
5789
5790         foreach my $key (keys %$cref) {
5791             my $value = $cref->{$key};
5792             if (is_valid_drivename($key)) {
5793                 next if $skip_drive && $key eq $skip_drive;
5794                 my $drive = parse_drive($key, $value);
5795                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5796                 return 1 if $volid eq $drive->{file};
5797                 if ($drive->{file} =~ m!^/!) {
5798                     return 1 if $drive->{file} eq $path;
5799                 } else {
5800                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5801                     next if !$storeid;
5802                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5803                     next if !$scfg;
5804                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5805                 }
5806             }
5807         }
5808
5809         return 0;
5810     };
5811
5812     return 1 if &$scan_config($conf);
5813
5814     undef $skip_drive;
5815
5816     foreach my $snapname (keys %{$conf->{snapshots}}) {
5817         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5818     }
5819
5820     return 0;
5821 }
5822
5823 sub update_disksize {
5824     my ($vmid, $conf, $volid_hash) = @_;
5825
5826     my $changes;
5827     my $prefix = "VM $vmid:";
5828
5829     # used and unused disks
5830     my $referenced = {};
5831
5832     # Note: it is allowed to define multiple storages with same path (alias), so
5833     # we need to check both 'volid' and real 'path' (two different volid can point
5834     # to the same path).
5835
5836     my $referencedpath = {};
5837
5838     # update size info
5839     foreach my $opt (keys %$conf) {
5840         if (is_valid_drivename($opt)) {
5841             my $drive = parse_drive($opt, $conf->{$opt});
5842             my $volid = $drive->{file};
5843             next if !$volid;
5844
5845             $referenced->{$volid} = 1;
5846             if ($volid_hash->{$volid} &&
5847                 (my $path = $volid_hash->{$volid}->{path})) {
5848                 $referencedpath->{$path} = 1;
5849             }
5850
5851             next if drive_is_cdrom($drive);
5852             next if !$volid_hash->{$volid};
5853
5854             $drive->{size} = $volid_hash->{$volid}->{size};
5855             my $new = print_drive($vmid, $drive);
5856             if ($new ne $conf->{$opt}) {
5857                 $changes = 1;
5858                 $conf->{$opt} = $new;
5859                 print "$prefix update disk '$opt' information.\n";
5860             }
5861         }
5862     }
5863
5864     # remove 'unusedX' entry if volume is used
5865     foreach my $opt (keys %$conf) {
5866         next if $opt !~ m/^unused\d+$/;
5867         my $volid = $conf->{$opt};
5868         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5869         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5870             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5871             $changes = 1;
5872             delete $conf->{$opt};
5873         }
5874
5875         $referenced->{$volid} = 1;
5876         $referencedpath->{$path} = 1 if $path;
5877     }
5878
5879     foreach my $volid (sort keys %$volid_hash) {
5880         next if $volid =~ m/vm-$vmid-state-/;
5881         next if $referenced->{$volid};
5882         my $path = $volid_hash->{$volid}->{path};
5883         next if !$path; # just to be sure
5884         next if $referencedpath->{$path};
5885         $changes = 1;
5886         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5887         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5888         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5889     }
5890
5891     return $changes;
5892 }
5893
5894 sub rescan {
5895     my ($vmid, $nolock, $dryrun) = @_;
5896
5897     my $cfg = PVE::Storage::config();
5898
5899     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5900     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5901     foreach my $stor (keys %{$cfg->{ids}}) {
5902         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5903     }
5904
5905     print "rescan volumes...\n";
5906     my $volid_hash = scan_volids($cfg, $vmid);
5907
5908     my $updatefn =  sub {
5909         my ($vmid) = @_;
5910
5911         my $conf = PVE::QemuConfig->load_config($vmid);
5912
5913         PVE::QemuConfig->check_lock($conf);
5914
5915         my $vm_volids = {};
5916         foreach my $volid (keys %$volid_hash) {
5917             my $info = $volid_hash->{$volid};
5918             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5919         }
5920
5921         my $changes = update_disksize($vmid, $conf, $vm_volids);
5922
5923         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5924     };
5925
5926     if (defined($vmid)) {
5927         if ($nolock) {
5928             &$updatefn($vmid);
5929         } else {
5930             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5931         }
5932     } else {
5933         my $vmlist = config_list();
5934         foreach my $vmid (keys %$vmlist) {
5935             if ($nolock) {
5936                 &$updatefn($vmid);
5937             } else {
5938                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5939             }
5940         }
5941     }
5942 }
5943
5944 sub restore_vma_archive {
5945     my ($archive, $vmid, $user, $opts, $comp) = @_;
5946
5947     my $readfrom = $archive;
5948
5949     my $cfg = PVE::Storage::config();
5950     my $commands = [];
5951     my $bwlimit = $opts->{bwlimit};
5952
5953     my $dbg_cmdstring = '';
5954     my $add_pipe = sub {
5955         my ($cmd) = @_;
5956         push @$commands, $cmd;
5957         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5958         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5959         $readfrom = '-';
5960     };
5961
5962     my $input = undef;
5963     if ($archive eq '-') {
5964         $input = '<&STDIN';
5965     } else {
5966         # If we use a backup from a PVE defined storage we also consider that
5967         # storage's rate limit:
5968         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5969         if (defined($volid)) {
5970             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5971             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5972             if ($readlimit) {
5973                 print STDERR "applying read rate limit: $readlimit\n";
5974                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5975                 $add_pipe->($cstream);
5976             }
5977         }
5978     }
5979
5980     if ($comp) {
5981         my $cmd;
5982         if ($comp eq 'gzip') {
5983             $cmd = ['zcat', $readfrom];
5984         } elsif ($comp eq 'lzop') {
5985             $cmd = ['lzop', '-d', '-c', $readfrom];
5986         } else {
5987             die "unknown compression method '$comp'\n";
5988         }
5989         $add_pipe->($cmd);
5990     }
5991
5992     my $tmpdir = "/var/tmp/vzdumptmp$$";
5993     rmtree $tmpdir;
5994
5995     # disable interrupts (always do cleanups)
5996     local $SIG{INT} =
5997         local $SIG{TERM} =
5998         local $SIG{QUIT} =
5999         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6000
6001     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6002     POSIX::mkfifo($mapfifo, 0600);
6003     my $fifofh;
6004
6005     my $openfifo = sub {
6006         open($fifofh, '>', $mapfifo) || die $!;
6007     };
6008
6009     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6010
6011     my $oldtimeout;
6012     my $timeout = 5;
6013
6014     my $devinfo = {};
6015
6016     my $rpcenv = PVE::RPCEnvironment::get();
6017
6018     my $conffile = PVE::QemuConfig->config_file($vmid);
6019     my $tmpfn = "$conffile.$$.tmp";
6020
6021     # Note: $oldconf is undef if VM does not exists
6022     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6023     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6024
6025     my %storage_limits;
6026
6027     my $print_devmap = sub {
6028         my $virtdev_hash = {};
6029
6030         my $cfgfn = "$tmpdir/qemu-server.conf";
6031
6032         # we can read the config - that is already extracted
6033         my $fh = IO::File->new($cfgfn, "r") ||
6034             "unable to read qemu-server.conf - $!\n";
6035
6036         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6037         if (-f $fwcfgfn) {
6038             my $pve_firewall_dir = '/etc/pve/firewall';
6039             mkdir $pve_firewall_dir; # make sure the dir exists
6040             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6041         }
6042
6043         while (defined(my $line = <$fh>)) {
6044             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6045                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6046                 die "archive does not contain data for drive '$virtdev'\n"
6047                     if !$devinfo->{$devname};
6048                 if (defined($opts->{storage})) {
6049                     $storeid = $opts->{storage} || 'local';
6050                 } elsif (!$storeid) {
6051                     $storeid = 'local';
6052                 }
6053                 $format = 'raw' if !$format;
6054                 $devinfo->{$devname}->{devname} = $devname;
6055                 $devinfo->{$devname}->{virtdev} = $virtdev;
6056                 $devinfo->{$devname}->{format} = $format;
6057                 $devinfo->{$devname}->{storeid} = $storeid;
6058
6059                 # check permission on storage
6060                 my $pool = $opts->{pool}; # todo: do we need that?
6061                 if ($user ne 'root@pam') {
6062                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6063                 }
6064
6065                 $storage_limits{$storeid} = $bwlimit;
6066
6067                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6068             }
6069         }
6070
6071         foreach my $key (keys %storage_limits) {
6072             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6073             next if !$limit;
6074             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6075             $storage_limits{$key} = $limit * 1024;
6076         }
6077
6078         foreach my $devname (keys %$devinfo) {
6079             die "found no device mapping information for device '$devname'\n"
6080                 if !$devinfo->{$devname}->{virtdev};
6081         }
6082
6083         # create empty/temp config
6084         if ($oldconf) {
6085             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6086             foreach_drive($oldconf, sub {
6087                 my ($ds, $drive) = @_;
6088
6089                 return if drive_is_cdrom($drive);
6090
6091                 my $volid = $drive->{file};
6092
6093                 return if !$volid || $volid =~ m|^/|;
6094
6095                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6096                 return if !$path || !$owner || ($owner != $vmid);
6097
6098                 # Note: only delete disk we want to restore
6099                 # other volumes will become unused
6100                 if ($virtdev_hash->{$ds}) {
6101                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6102                     if (my $err = $@) {
6103                         warn $err;
6104                     }
6105                 }
6106             });
6107
6108             # delete vmstate files
6109             # since after the restore we have no snapshots anymore
6110             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6111                 my $snap = $oldconf->{snapshots}->{$snapname};
6112                 if ($snap->{vmstate}) {
6113                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6114                     if (my $err = $@) {
6115                         warn $err;
6116                     }
6117                 }
6118             }
6119         }
6120
6121         my $map = {};
6122         foreach my $virtdev (sort keys %$virtdev_hash) {
6123             my $d = $virtdev_hash->{$virtdev};
6124             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6125             my $storeid = $d->{storeid};
6126             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6127
6128             my $map_opts = '';
6129             if (my $limit = $storage_limits{$storeid}) {
6130                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6131             }
6132
6133             # test if requested format is supported
6134             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6135             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6136             $d->{format} = $defFormat if !$supported;
6137
6138             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6139                                                   $d->{format}, undef, $alloc_size);
6140             print STDERR "new volume ID is '$volid'\n";
6141             $d->{volid} = $volid;
6142             my $path = PVE::Storage::path($cfg, $volid);
6143
6144             PVE::Storage::activate_volumes($cfg,[$volid]);
6145
6146             my $write_zeros = 1;
6147             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6148                 $write_zeros = 0;
6149             }
6150
6151             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6152
6153             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6154             $map->{$virtdev} = $volid;
6155         }
6156
6157         $fh->seek(0, 0) || die "seek failed - $!\n";
6158
6159         my $outfd = new IO::File ($tmpfn, "w") ||
6160             die "unable to write config for VM $vmid\n";
6161
6162         my $cookie = { netcount => 0 };
6163         while (defined(my $line = <$fh>)) {
6164             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6165         }
6166
6167         $fh->close();
6168         $outfd->close();
6169     };
6170
6171     eval {
6172         # enable interrupts
6173         local $SIG{INT} =
6174             local $SIG{TERM} =
6175             local $SIG{QUIT} =
6176             local $SIG{HUP} =
6177             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6178         local $SIG{ALRM} = sub { die "got timeout\n"; };
6179
6180         $oldtimeout = alarm($timeout);
6181
6182         my $parser = sub {
6183             my $line = shift;
6184
6185             print "$line\n";
6186
6187             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6188                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6189                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6190             } elsif ($line =~ m/^CTIME: /) {
6191                 # we correctly received the vma config, so we can disable
6192                 # the timeout now for disk allocation (set to 10 minutes, so
6193                 # that we always timeout if something goes wrong)
6194                 alarm(600);
6195                 &$print_devmap();
6196                 print $fifofh "done\n";
6197                 my $tmp = $oldtimeout || 0;
6198                 $oldtimeout = undef;
6199                 alarm($tmp);
6200                 close($fifofh);
6201             }
6202         };
6203
6204         print "restore vma archive: $dbg_cmdstring\n";
6205         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6206     };
6207     my $err = $@;
6208
6209     alarm($oldtimeout) if $oldtimeout;
6210
6211     my $vollist = [];
6212     foreach my $devname (keys %$devinfo) {
6213         my $volid = $devinfo->{$devname}->{volid};
6214         push @$vollist, $volid if $volid;
6215     }
6216
6217     PVE::Storage::deactivate_volumes($cfg, $vollist);
6218
6219     unlink $mapfifo;
6220
6221     if ($err) {
6222         rmtree $tmpdir;
6223         unlink $tmpfn;
6224
6225         foreach my $devname (keys %$devinfo) {
6226             my $volid = $devinfo->{$devname}->{volid};
6227             next if !$volid;
6228             eval {
6229                 if ($volid =~ m|^/|) {
6230                     unlink $volid || die 'unlink failed\n';
6231                 } else {
6232                     PVE::Storage::vdisk_free($cfg, $volid);
6233                 }
6234                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6235             };
6236             print STDERR "unable to cleanup '$volid' - $@" if $@;
6237         }
6238         die $err;
6239     }
6240
6241     rmtree $tmpdir;
6242
6243     rename($tmpfn, $conffile) ||
6244         die "unable to commit configuration file '$conffile'\n";
6245
6246     PVE::Cluster::cfs_update(); # make sure we read new file
6247
6248     eval { rescan($vmid, 1); };
6249     warn $@ if $@;
6250 }
6251
6252 sub restore_tar_archive {
6253     my ($archive, $vmid, $user, $opts) = @_;
6254
6255     if ($archive ne '-') {
6256         my $firstfile = tar_archive_read_firstfile($archive);
6257         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6258             if $firstfile ne 'qemu-server.conf';
6259     }
6260
6261     my $storecfg = PVE::Storage::config();
6262
6263     # destroy existing data - keep empty config
6264     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6265     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6266
6267     my $tocmd = "/usr/lib/qemu-server/qmextract";
6268
6269     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6270     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6271     $tocmd .= ' --prealloc' if $opts->{prealloc};
6272     $tocmd .= ' --info' if $opts->{info};
6273
6274     # tar option "xf" does not autodetect compression when read from STDIN,
6275     # so we pipe to zcat
6276     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6277         PVE::Tools::shellquote("--to-command=$tocmd");
6278
6279     my $tmpdir = "/var/tmp/vzdumptmp$$";
6280     mkpath $tmpdir;
6281
6282     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6283     local $ENV{VZDUMP_VMID} = $vmid;
6284     local $ENV{VZDUMP_USER} = $user;
6285
6286     my $conffile = PVE::QemuConfig->config_file($vmid);
6287     my $tmpfn = "$conffile.$$.tmp";
6288
6289     # disable interrupts (always do cleanups)
6290     local $SIG{INT} =
6291         local $SIG{TERM} =
6292         local $SIG{QUIT} =
6293         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6294
6295     eval {
6296         # enable interrupts
6297         local $SIG{INT} =
6298             local $SIG{TERM} =
6299             local $SIG{QUIT} =
6300             local $SIG{HUP} =
6301             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6302
6303         if ($archive eq '-') {
6304             print "extracting archive from STDIN\n";
6305             run_command($cmd, input => "<&STDIN");
6306         } else {
6307             print "extracting archive '$archive'\n";
6308             run_command($cmd);
6309         }
6310
6311         return if $opts->{info};
6312
6313         # read new mapping
6314         my $map = {};
6315         my $statfile = "$tmpdir/qmrestore.stat";
6316         if (my $fd = IO::File->new($statfile, "r")) {
6317             while (defined (my $line = <$fd>)) {
6318                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6319                     $map->{$1} = $2 if $1;
6320                 } else {
6321                     print STDERR "unable to parse line in statfile - $line\n";
6322                 }
6323             }
6324             $fd->close();
6325         }
6326
6327         my $confsrc = "$tmpdir/qemu-server.conf";
6328
6329         my $srcfd = new IO::File($confsrc, "r") ||
6330             die "unable to open file '$confsrc'\n";
6331
6332         my $outfd = new IO::File ($tmpfn, "w") ||
6333             die "unable to write config for VM $vmid\n";
6334
6335         my $cookie = { netcount => 0 };
6336         while (defined (my $line = <$srcfd>)) {
6337             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6338         }
6339
6340         $srcfd->close();
6341         $outfd->close();
6342     };
6343     my $err = $@;
6344
6345     if ($err) {
6346
6347         unlink $tmpfn;
6348
6349         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6350
6351         die $err;
6352     }
6353
6354     rmtree $tmpdir;
6355
6356     rename $tmpfn, $conffile ||
6357         die "unable to commit configuration file '$conffile'\n";
6358
6359     PVE::Cluster::cfs_update(); # make sure we read new file
6360
6361     eval { rescan($vmid, 1); };
6362     warn $@ if $@;
6363 };
6364
6365 sub foreach_storage_used_by_vm {
6366     my ($conf, $func) = @_;
6367
6368     my $sidhash = {};
6369
6370     foreach_drive($conf, sub {
6371         my ($ds, $drive) = @_;
6372         return if drive_is_cdrom($drive);
6373
6374         my $volid = $drive->{file};
6375
6376         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6377         $sidhash->{$sid} = $sid if $sid;
6378     });
6379
6380     foreach my $sid (sort keys %$sidhash) {
6381         &$func($sid);
6382     }
6383 }
6384
6385 sub do_snapshots_with_qemu {
6386     my ($storecfg, $volid) = @_;
6387
6388     my $storage_name = PVE::Storage::parse_volume_id($volid);
6389
6390     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6391         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6392         return 1;
6393     }
6394
6395     if ($volid =~ m/\.(qcow2|qed)$/){
6396         return 1;
6397     }
6398
6399     return undef;
6400 }
6401
6402 sub qga_check_running {
6403     my ($vmid, $nowarn) = @_;
6404
6405     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6406     if ($@) {
6407         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6408         return 0;
6409     }
6410     return 1;
6411 }
6412
6413 sub template_create {
6414     my ($vmid, $conf, $disk) = @_;
6415
6416     my $storecfg = PVE::Storage::config();
6417
6418     foreach_drive($conf, sub {
6419         my ($ds, $drive) = @_;
6420
6421         return if drive_is_cdrom($drive);
6422         return if $disk && $ds ne $disk;
6423
6424         my $volid = $drive->{file};
6425         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6426
6427         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6428         $drive->{file} = $voliddst;
6429         $conf->{$ds} = print_drive($vmid, $drive);
6430         PVE::QemuConfig->write_config($vmid, $conf);
6431     });
6432 }
6433
6434 sub qemu_img_convert {
6435     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6436
6437     my $storecfg = PVE::Storage::config();
6438     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6439     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6440
6441     if ($src_storeid && $dst_storeid) {
6442
6443         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6444
6445         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6446         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6447
6448         my $src_format = qemu_img_format($src_scfg, $src_volname);
6449         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6450
6451         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6452         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6453
6454         my $cmd = [];
6455         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6456         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6457         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6458         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6459         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6460         if ($is_zero_initialized) {
6461             push @$cmd, "zeroinit:$dst_path";
6462         } else {
6463             push @$cmd, $dst_path;
6464         }
6465
6466         my $parser = sub {
6467             my $line = shift;
6468             if($line =~ m/\((\S+)\/100\%\)/){
6469                 my $percent = $1;
6470                 my $transferred = int($size * $percent / 100);
6471                 my $remaining = $size - $transferred;
6472
6473                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6474             }
6475
6476         };
6477
6478         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6479         my $err = $@;
6480         die "copy failed: $err" if $err;
6481     }
6482 }
6483
6484 sub qemu_img_format {
6485     my ($scfg, $volname) = @_;
6486
6487     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6488         return $1;
6489     } else {
6490         return "raw";
6491     }
6492 }
6493
6494 sub qemu_drive_mirror {
6495     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6496
6497     $jobs = {} if !$jobs;
6498
6499     my $qemu_target;
6500     my $format;
6501     $jobs->{"drive-$drive"} = {};
6502
6503     if ($dst_volid =~ /^nbd:/) {
6504         $qemu_target = $dst_volid;
6505         $format = "nbd";
6506     } else {
6507         my $storecfg = PVE::Storage::config();
6508         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6509
6510         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6511
6512         $format = qemu_img_format($dst_scfg, $dst_volname);
6513
6514         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6515
6516         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6517     }
6518
6519     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6520     $opts->{format} = $format if $format;
6521
6522     print "drive mirror is starting for drive-$drive\n";
6523
6524     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6525
6526     if (my $err = $@) {
6527         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6528         die "mirroring error: $err";
6529     }
6530
6531     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6532 }
6533
6534 sub qemu_drive_mirror_monitor {
6535     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6536
6537     eval {
6538         my $err_complete = 0;
6539
6540         while (1) {
6541             die "storage migration timed out\n" if $err_complete > 300;
6542
6543             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6544
6545             my $running_mirror_jobs = {};
6546             foreach my $stat (@$stats) {
6547                 next if $stat->{type} ne 'mirror';
6548                 $running_mirror_jobs->{$stat->{device}} = $stat;
6549             }
6550
6551             my $readycounter = 0;
6552
6553             foreach my $job (keys %$jobs) {
6554
6555                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6556                     print "$job : finished\n";
6557                     delete $jobs->{$job};
6558                     next;
6559                 }
6560
6561                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6562
6563                 my $busy = $running_mirror_jobs->{$job}->{busy};
6564                 my $ready = $running_mirror_jobs->{$job}->{ready};
6565                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6566                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6567                     my $remaining = $total - $transferred;
6568                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6569
6570                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6571                 }
6572
6573                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6574             }
6575
6576             last if scalar(keys %$jobs) == 0;
6577
6578             if ($readycounter == scalar(keys %$jobs)) {
6579                 print "all mirroring jobs are ready \n";
6580                 last if $skipcomplete; #do the complete later
6581
6582                 if ($vmiddst && $vmiddst != $vmid) {
6583                     my $agent_running = $qga && qga_check_running($vmid);
6584                     if ($agent_running) {
6585                         print "freeze filesystem\n";
6586                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6587                     } else {
6588                         print "suspend vm\n";
6589                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6590                     }
6591
6592                     # if we clone a disk for a new target vm, we don't switch the disk
6593                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6594
6595                     if ($agent_running) {
6596                         print "unfreeze filesystem\n";
6597                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6598                     } else {
6599                         print "resume vm\n";
6600                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6601                     }
6602
6603                     last;
6604                 } else {
6605
6606                     foreach my $job (keys %$jobs) {
6607                         # try to switch the disk if source and destination are on the same guest
6608                         print "$job: Completing block job...\n";
6609
6610                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6611                         if ($@ =~ m/cannot be completed/) {
6612                             print "$job: Block job cannot be completed, try again.\n";
6613                             $err_complete++;
6614                         }else {
6615                             print "$job: Completed successfully.\n";
6616                             $jobs->{$job}->{complete} = 1;
6617                         }
6618                     }
6619                 }
6620             }
6621             sleep 1;
6622         }
6623     };
6624     my $err = $@;
6625
6626     if ($err) {
6627         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6628         die "mirroring error: $err";
6629     }
6630
6631 }
6632
6633 sub qemu_blockjobs_cancel {
6634     my ($vmid, $jobs) = @_;
6635
6636     foreach my $job (keys %$jobs) {
6637         print "$job: Cancelling block job\n";
6638         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6639         $jobs->{$job}->{cancel} = 1;
6640     }
6641
6642     while (1) {
6643         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6644
6645         my $running_jobs = {};
6646         foreach my $stat (@$stats) {
6647             $running_jobs->{$stat->{device}} = $stat;
6648         }
6649
6650         foreach my $job (keys %$jobs) {
6651
6652             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6653                 print "$job: Done.\n";
6654                 delete $jobs->{$job};
6655             }
6656         }
6657
6658         last if scalar(keys %$jobs) == 0;
6659
6660         sleep 1;
6661     }
6662 }
6663
6664 sub clone_disk {
6665     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6666         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6667
6668     my $newvolid;
6669
6670     if (!$full) {
6671         print "create linked clone of drive $drivename ($drive->{file})\n";
6672         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6673         push @$newvollist, $newvolid;
6674     } else {
6675
6676         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6677         $storeid = $storage if $storage;
6678
6679         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6680         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6681
6682         print "create full clone of drive $drivename ($drive->{file})\n";
6683         my $name = undef;
6684         if (drive_is_cloudinit($drive)) {
6685             $name = "vm-$newvmid-cloudinit";
6686             $snapname = undef;
6687             # cloudinit only supports raw and qcow2 atm:
6688             if ($dst_format eq 'qcow2') {
6689                 $name .= '.qcow2';
6690             } elsif ($dst_format ne 'raw') {
6691                 die "clone: unhandled format for cloudinit image\n";
6692             }
6693         }
6694         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6695         push @$newvollist, $newvolid;
6696
6697         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6698
6699         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6700         if (!$running || $snapname) {
6701             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6702         } else {
6703
6704             my $kvmver = get_running_qemu_version ($vmid);
6705             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6706                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6707                     if $drive->{iothread};
6708             }
6709
6710             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6711         }
6712     }
6713
6714     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6715
6716     my $disk = $drive;
6717     $disk->{format} = undef;
6718     $disk->{file} = $newvolid;
6719     $disk->{size} = $size;
6720
6721     return $disk;
6722 }
6723
6724 # this only works if VM is running
6725 sub get_current_qemu_machine {
6726     my ($vmid) = @_;
6727
6728     my $cmd = { execute => 'query-machines', arguments => {} };
6729     my $res = vm_qmp_command($vmid, $cmd);
6730
6731     my ($current, $default);
6732     foreach my $e (@$res) {
6733         $default = $e->{name} if $e->{'is-default'};
6734         $current = $e->{name} if $e->{'is-current'};
6735     }
6736
6737     # fallback to the default machine if current is not supported by qemu
6738     return $current || $default || 'pc';
6739 }
6740
6741 sub get_running_qemu_version {
6742     my ($vmid) = @_;
6743     my $cmd = { execute => 'query-version', arguments => {} };
6744     my $res = vm_qmp_command($vmid, $cmd);
6745     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6746 }
6747
6748 sub qemu_machine_feature_enabled {
6749     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6750
6751     my $current_major;
6752     my $current_minor;
6753
6754     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6755
6756         $current_major = $3;
6757         $current_minor = $4;
6758
6759     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6760
6761         $current_major = $1;
6762         $current_minor = $2;
6763     }
6764
6765     return 1 if $current_major > $version_major ||
6766                 ($current_major == $version_major &&
6767                  $current_minor >= $version_minor);
6768 }
6769
6770 sub qemu_machine_pxe {
6771     my ($vmid, $conf, $machine) = @_;
6772
6773     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6774
6775     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6776         $machine .= '.pxe';
6777     }
6778
6779     return $machine;
6780 }
6781
6782 sub qemu_use_old_bios_files {
6783     my ($machine_type) = @_;
6784
6785     return if !$machine_type;
6786
6787     my $use_old_bios_files = undef;
6788
6789     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6790         $machine_type = $1;
6791         $use_old_bios_files = 1;
6792     } else {
6793         my $kvmver = kvm_user_version();
6794         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6795         # load new efi bios files on migration. So this hack is required to allow
6796         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6797         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6798         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6799     }
6800
6801     return ($use_old_bios_files, $machine_type);
6802 }
6803
6804 sub create_efidisk($$$$$) {
6805     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6806
6807     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6808     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6809
6810     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6811     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6812     PVE::Storage::activate_volumes($storecfg, [$volid]);
6813
6814     my $path = PVE::Storage::path($storecfg, $volid);
6815     eval {
6816         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6817     };
6818     die "Copying EFI vars image failed: $@" if $@;
6819
6820     return ($volid, $vars_size);
6821 }
6822
6823 sub vm_iothreads_list {
6824     my ($vmid) = @_;
6825
6826     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6827
6828     my $iothreads = {};
6829     foreach my $iothread (@$res) {
6830         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6831     }
6832
6833     return $iothreads;
6834 }
6835
6836 sub scsihw_infos {
6837     my ($conf, $drive) = @_;
6838
6839     my $maxdev = 0;
6840
6841     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6842         $maxdev = 7;
6843     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6844         $maxdev = 1;
6845     } else {
6846         $maxdev = 256;
6847     }
6848
6849     my $controller = int($drive->{index} / $maxdev);
6850     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6851
6852     return ($maxdev, $controller, $controller_prefix);
6853 }
6854
6855 sub add_hyperv_enlightenments {
6856     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
6857
6858     return if $winversion < 6;
6859     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6860
6861     if ($gpu_passthrough || defined($hv_vendor_id)) {
6862         $hv_vendor_id //= 'proxmox';
6863         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
6864     }
6865
6866     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6867         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6868         push @$cpuFlags , 'hv_vapic';
6869         push @$cpuFlags , 'hv_time';
6870     } else {
6871         push @$cpuFlags , 'hv_spinlocks=0xffff';
6872     }
6873
6874     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6875         push @$cpuFlags , 'hv_reset';
6876         push @$cpuFlags , 'hv_vpindex';
6877         push @$cpuFlags , 'hv_runtime';
6878     }
6879
6880     if ($winversion >= 7) {
6881         push @$cpuFlags , 'hv_relaxed';
6882
6883         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6884             push @$cpuFlags , 'hv_synic';
6885             push @$cpuFlags , 'hv_stimer';
6886         }
6887     }
6888 }
6889
6890 sub windows_version {
6891     my ($ostype) = @_;
6892
6893     return 0 if !$ostype;
6894
6895     my $winversion = 0;
6896
6897     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6898         $winversion = 5;
6899     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6900         $winversion = 6;
6901     } elsif ($ostype =~ m/^win(\d+)$/) {
6902         $winversion = $1;
6903     }
6904
6905     return $winversion;
6906 }
6907
6908 sub resolve_dst_disk_format {
6909         my ($storecfg, $storeid, $src_volname, $format) = @_;
6910         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6911
6912         if (!$format) {
6913             # if no target format is specified, use the source disk format as hint
6914             if ($src_volname) {
6915                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6916                 $format = qemu_img_format($scfg, $src_volname);
6917             } else {
6918                 return $defFormat;
6919             }
6920         }
6921
6922         # test if requested format is supported - else use default
6923         my $supported = grep { $_ eq $format } @$validFormats;
6924         $format = $defFormat if !$supported;
6925         return $format;
6926 }
6927
6928 sub resolve_first_disk {
6929     my $conf = shift;
6930     my @disks = PVE::QemuServer::valid_drive_names();
6931     my $firstdisk;
6932     foreach my $ds (reverse @disks) {
6933         next if !$conf->{$ds};
6934         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6935         next if PVE::QemuServer::drive_is_cdrom($disk);
6936         $firstdisk = $ds;
6937     }
6938     return $firstdisk;
6939 }
6940
6941 sub generate_uuid {
6942     my ($uuid, $uuid_str);
6943     UUID::generate($uuid);
6944     UUID::unparse($uuid, $uuid_str);
6945     return $uuid_str;
6946 }
6947
6948 sub generate_smbios1_uuid {
6949     return "uuid=".generate_uuid();
6950 }
6951
6952 sub nbd_stop {
6953     my ($vmid) = @_;
6954
6955     vm_mon_cmd($vmid, 'nbd-server-stop');
6956 }
6957
6958 # bash completion helper
6959
6960 sub complete_backup_archives {
6961     my ($cmdname, $pname, $cvalue) = @_;
6962
6963     my $cfg = PVE::Storage::config();
6964
6965     my $storeid;
6966
6967     if ($cvalue =~ m/^([^:]+):/) {
6968         $storeid = $1;
6969     }
6970
6971     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6972
6973     my $res = [];
6974     foreach my $id (keys %$data) {
6975         foreach my $item (@{$data->{$id}}) {
6976             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6977             push @$res, $item->{volid} if defined($item->{volid});
6978         }
6979     }
6980
6981     return $res;
6982 }
6983
6984 my $complete_vmid_full = sub {
6985     my ($running) = @_;
6986
6987     my $idlist = vmstatus();
6988
6989     my $res = [];
6990
6991     foreach my $id (keys %$idlist) {
6992         my $d = $idlist->{$id};
6993         if (defined($running)) {
6994             next if $d->{template};
6995             next if $running && $d->{status} ne 'running';
6996             next if !$running && $d->{status} eq 'running';
6997         }
6998         push @$res, $id;
6999
7000     }
7001     return $res;
7002 };
7003
7004 sub complete_vmid {
7005     return &$complete_vmid_full();
7006 }
7007
7008 sub complete_vmid_stopped {
7009     return &$complete_vmid_full(0);
7010 }
7011
7012 sub complete_vmid_running {
7013     return &$complete_vmid_full(1);
7014 }
7015
7016 sub complete_storage {
7017
7018     my $cfg = PVE::Storage::config();
7019     my $ids = $cfg->{ids};
7020
7021     my $res = [];
7022     foreach my $sid (keys %$ids) {
7023         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7024         next if !$ids->{$sid}->{content}->{images};
7025         push @$res, $sid;
7026     }
7027
7028     return $res;
7029 }
7030
7031 1;