PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $confdesc = {
 260     onboot => {
 261         optional => 1,
 262         type => 'boolean',
 263         description => "Specifies whether a VM will be started during system bootup.",
 264         default => 0,
 265     },
 266     autostart => {
 267         optional => 1,
 268         type => 'boolean',
 269         description => "Automatic restart after crash (currently ignored).",
 270         default => 0,
 271     },
 272     hotplug => {
 273         optional => 1,
 274         type => 'string', format => 'pve-hotplug-features',
 275         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 276         default => 'network,disk,usb',
 277     },
 278     reboot => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 282         default => 1,
 283     },
 284     lock => {
 285         optional => 1,
 286         type => 'string',
 287         description => "Lock/unlock the VM.",
 288         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 289     },
 290     cpulimit => {
 291         optional => 1,
 292         type => 'number',
 293         description => "Limit of CPU usage.",
 294         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 295         minimum => 0,
 296         maximum => 128,
 297         default => 0,
 298     },
 299     cpuunits => {
 300         optional => 1,
 301         type => 'integer',
 302         description => "CPU weight for a VM.",
 303         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 304         minimum => 2,
 305         maximum => 262144,
 306         default => 1024,
 307     },
 308     memory => {
 309         optional => 1,
 310         type => 'integer',
 311         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 312         minimum => 16,
 313         default => 512,
 314     },
 315     balloon => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 319         minimum => 0,
 320     },
 321     shares => {
 322         optional => 1,
 323         type => 'integer',
 324         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 325         minimum => 0,
 326         maximum => 50000,
 327         default => 1000,
 328     },
 329     keyboard => {
 330         optional => 1,
 331         type => 'string',
 332         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 333                        "It should not be necessary to set it.",
 334         enum => PVE::Tools::kvmkeymaplist(),
 335         default => undef,
 336     },
 337     name => {
 338         optional => 1,
 339         type => 'string', format => 'dns-name',
 340         description => "Set a name for the VM. Only used on the configuration web interface.",
 341     },
 342     scsihw => {
 343         optional => 1,
 344         type => 'string',
 345         description => "SCSI controller model",
 346         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 347         default => 'lsi',
 348     },
 349     description => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 353     },
 354     ostype => {
 355         optional => 1,
 356         type => 'string',
 357         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 358         description => "Specify guest operating system.",
 359         verbose_description => <<EODESC,
 360 Specify guest operating system. This is used to enable special
 361 optimization/features for specific operating systems:
 362
 363 [horizontal]
 364 other;; unspecified OS
 365 wxp;; Microsoft Windows XP
 366 w2k;; Microsoft Windows 2000
 367 w2k3;; Microsoft Windows 2003
 368 w2k8;; Microsoft Windows 2008
 369 wvista;; Microsoft Windows Vista
 370 win7;; Microsoft Windows 7
 371 win8;; Microsoft Windows 8/2012/2012r2
 372 win10;; Microsoft Windows 10/2016
 373 l24;; Linux 2.4 Kernel
 374 l26;; Linux 2.6/3.X Kernel
 375 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 376 EODESC
 377     },
 378     boot => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 382         pattern => '[acdn]{1,4}',
 383         default => 'cdn',
 384     },
 385     bootdisk => {
 386         optional => 1,
 387         type => 'string', format => 'pve-qm-bootdisk',
 388         description => "Enable booting from specified disk.",
 389         pattern => '(ide|sata|scsi|virtio)\d+',
 390     },
 391     smp => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "The number of CPUs. Please use option -sockets instead.",
 395         minimum => 1,
 396         default => 1,
 397     },
 398     sockets => {
 399         optional => 1,
 400         type => 'integer',
 401         description => "The number of CPU sockets.",
 402         minimum => 1,
 403         default => 1,
 404     },
 405     cores => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of cores per socket.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     numa => {
 413         optional => 1,
 414         type => 'boolean',
 415         description => "Enable/disable NUMA.",
 416         default => 0,
 417     },
 418     hugepages => {
 419         optional => 1,
 420         type => 'string',
 421         description => "Enable/disable hugepages memory.",
 422         enum => [qw(any 2 1024)],
 423     },
 424     vcpus => {
 425         optional => 1,
 426         type => 'integer',
 427         description => "Number of hotplugged vcpus.",
 428         minimum => 1,
 429         default => 0,
 430     },
 431     acpi => {
 432         optional => 1,
 433         type => 'boolean',
 434         description => "Enable/disable ACPI.",
 435         default => 1,
 436     },
 437     agent => {
 438         optional => 1,
 439         description => "Enable/disable Qemu GuestAgent and its properties.",
 440         type => 'string',
 441         format => $agent_fmt,
 442     },
 443     kvm => {
 444         optional => 1,
 445         type => 'boolean',
 446         description => "Enable/disable KVM hardware virtualization.",
 447         default => 1,
 448     },
 449     tdf => {
 450         optional => 1,
 451         type => 'boolean',
 452         description => "Enable/disable time drift fix.",
 453         default => 0,
 454     },
 455     localtime => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 459     },
 460     freeze => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 464     },
 465     vga => {
 466         optional => 1,
 467         type => 'string', format => $vga_fmt,
 468         description => "Configure the VGA hardware.",
 469         verbose_description => "Configure the VGA Hardware. If you want to use ".
 470             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 471             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 472             "is 'std' for all OS types besides some Windows versions (XP and " .
 473             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 474             "display server. For win* OS you can select how many independent " .
 475             "displays you want, Linux guests can add displays them self.\n".
 476             "You can also run without any graphic card, using a serial device as terminal.",
 477     },
 478     watchdog => {
 479         optional => 1,
 480         type => 'string', format => 'pve-qm-watchdog',
 481         description => "Create a virtual hardware watchdog device.",
 482         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 483             " (by a guest action), the watchdog must be periodically polled " .
 484             "by an agent inside the guest or else the watchdog will reset " .
 485             "the guest (or execute the respective action specified)",
 486     },
 487     startdate => {
 488         optional => 1,
 489         type => 'string',
 490         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 491         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 492         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 493         default => 'now',
 494     },
 495     startup =>  get_standard_option('pve-startup-order'),
 496     template => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable Template.",
 500         default => 0,
 501     },
 502     args => {
 503         optional => 1,
 504         type => 'string',
 505         description => "Arbitrary arguments passed to kvm.",
 506         verbose_description => <<EODESCR,
 507 Arbitrary arguments passed to kvm, for example:
 508
 509 args: -no-reboot -no-hpet
 510
 511 NOTE: this option is for experts only.
 512 EODESCR
 513     },
 514     tablet => {
 515         optional => 1,
 516         type => 'boolean',
 517         default => 1,
 518         description => "Enable/disable the USB tablet device.",
 519         verbose_description => "Enable/disable the USB tablet device. This device is " .
 520             "usually needed to allow absolute mouse positioning with VNC. " .
 521             "Else the mouse runs out of sync with normal VNC clients. " .
 522             "If you're running lots of console-only guests on one host, " .
 523             "you may consider disabling this to save some context switches. " .
 524             "This is turned off by default if you use spice (-vga=qxl).",
 525     },
 526     migrate_speed => {
 527         optional => 1,
 528         type => 'integer',
 529         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 530         minimum => 0,
 531         default => 0,
 532     },
 533     migrate_downtime => {
 534         optional => 1,
 535         type => 'number',
 536         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 537         minimum => 0,
 538         default => 0.1,
 539     },
 540     cdrom => {
 541         optional => 1,
 542         type => 'string', format => 'pve-qm-ide',
 543         typetext => '<volume>',
 544         description => "This is an alias for option -ide2",
 545     },
 546     cpu => {
 547         optional => 1,
 548         description => "Emulated CPU type.",
 549         type => 'string',
 550         format => $cpu_fmt,
 551     },
 552     parent => get_standard_option('pve-snapshot-name', {
 553         optional => 1,
 554         description => "Parent snapshot name. This is used internally, and should not be modified.",
 555     }),
 556     snaptime => {
 557         optional => 1,
 558         description => "Timestamp for snapshots.",
 559         type => 'integer',
 560         minimum => 0,
 561     },
 562     vmstate => {
 563         optional => 1,
 564         type => 'string', format => 'pve-volume-id',
 565         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 566     },
 567     vmstatestorage => get_standard_option('pve-storage-id', {
 568         description => "Default storage for VM state volumes/files.",
 569         optional => 1,
 570     }),
 571     runningmachine => get_standard_option('pve-qemu-machine', {
 572         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 573     }),
 574     machine => get_standard_option('pve-qemu-machine'),
 575     arch => {
 576         description => "Virtual processor architecture. Defaults to the host.",
 577         optional => 1,
 578         type => 'string',
 579         enum => [qw(x86_64 aarch64)],
 580     },
 581     smbios1 => {
 582         description => "Specify SMBIOS type 1 fields.",
 583         type => 'string', format => 'pve-qm-smbios1',
 584         maxLength => 256,
 585         optional => 1,
 586     },
 587     protection => {
 588         optional => 1,
 589         type => 'boolean',
 590         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 591         default => 0,
 592     },
 593     bios => {
 594         optional => 1,
 595         type => 'string',
 596         enum => [ qw(seabios ovmf) ],
 597         description => "Select BIOS implementation.",
 598         default => 'seabios',
 599     },
 600     vmgenid => {
 601         type => 'string',
 602         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 603         format_description => 'UUID',
 604         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 605         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 606             " 128-bit integer value identifier to the guest OS. This allows to".
 607             " notify the guest operating system when the virtual machine is".
 608             " executed with a different configuration (e.g. snapshot execution".
 609             " or creation from a template). The guest operating system notices".
 610             " the change, and is then able to react as appropriate by marking".
 611             " its copies of distributed databases as dirty, re-initializing its".
 612             " random number generator, etc.\n".
 613             "Note that auto-creation only works when done throug API/CLI create".
 614             " or update methods, but not when manually editing the config file.",
 615         default => "1 (autogenerated)",
 616         optional => 1,
 617     },
 618     hookscript => {
 619         type => 'string',
 620         format => 'pve-volume-id',
 621         optional => 1,
 622         description => "Script that will be executed during various steps in the vms lifetime.",
 623     },
 624 };
 625
 626 my $confdesc_cloudinit = {
 627     citype => {
 628         optional => 1,
 629         type => 'string',
 630         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 631         enum => ['configdrive2', 'nocloud'],
 632     },
 633     ciuser => {
 634         optional => 1,
 635         type => 'string',
 636         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 637     },
 638     cipassword => {
 639         optional => 1,
 640         type => 'string',
 641         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 642     },
 643     searchdomain => {
 644         optional => 1,
 645         type => 'string',
 646         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 647     },
 648     nameserver => {
 649         optional => 1,
 650         type => 'string', format => 'address-list',
 651         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 652     },
 653     sshkeys => {
 654         optional => 1,
 655         type => 'string',
 656         format => 'urlencoded',
 657         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 658     },
 659 };
 660
 661 # what about other qemu settings ?
 662 #cpu => 'string',
 663 #machine => 'string',
 664 #fda => 'file',
 665 #fdb => 'file',
 666 #mtdblock => 'file',
 667 #sd => 'file',
 668 #pflash => 'file',
 669 #snapshot => 'bool',
 670 #bootp => 'file',
 671 ##tftp => 'dir',
 672 ##smb => 'dir',
 673 #kernel => 'file',
 674 #append => 'string',
 675 #initrd => 'file',
 676 ##soundhw => 'string',
 677
 678 while (my ($k, $v) = each %$confdesc) {
 679     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 680 }
 681
 682 my $MAX_IDE_DISKS = 4;
 683 my $MAX_SCSI_DISKS = 14;
 684 my $MAX_VIRTIO_DISKS = 16;
 685 my $MAX_SATA_DISKS = 6;
 686 my $MAX_USB_DEVICES = 5;
 687 my $MAX_NETS = 32;
 688 my $MAX_UNUSED_DISKS = 256;
 689 my $MAX_HOSTPCI_DEVICES = 4;
 690 my $MAX_SERIAL_PORTS = 4;
 691 my $MAX_PARALLEL_PORTS = 3;
 692 my $MAX_NUMA = 8;
 693
 694 my $numa_fmt = {
 695     cpus => {
 696         type => "string",
 697         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 698         description => "CPUs accessing this NUMA node.",
 699         format_description => "id[-id];...",
 700     },
 701     memory => {
 702         type => "number",
 703         description => "Amount of memory this NUMA node provides.",
 704         optional => 1,
 705     },
 706     hostnodes => {
 707         type => "string",
 708         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 709         description => "Host NUMA nodes to use.",
 710         format_description => "id[-id];...",
 711         optional => 1,
 712     },
 713     policy => {
 714         type => 'string',
 715         enum => [qw(preferred bind interleave)],
 716         description => "NUMA allocation policy.",
 717         optional => 1,
 718     },
 719 };
 720 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 721 my $numadesc = {
 722     optional => 1,
 723     type => 'string', format => $numa_fmt,
 724     description => "NUMA topology.",
 725 };
 726 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 727
 728 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 729     $confdesc->{"numa$i"} = $numadesc;
 730 }
 731
 732 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 733                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 734                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 735 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 736
 737 my $net_fmt_bridge_descr = <<__EOD__;
 738 Bridge to attach the network device to. The Proxmox VE standard bridge
 739 is called 'vmbr0'.
 740
 741 If you do not specify a bridge, we create a kvm user (NATed) network
 742 device, which provides DHCP and DNS services. The following addresses
 743 are used:
 744
 745  10.0.2.2   Gateway
 746  10.0.2.3   DNS Server
 747  10.0.2.4   SMB Server
 748
 749 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 750 __EOD__
 751
 752 my $net_fmt = {
 753     macaddr => {
 754         type => 'string',
 755         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 756         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 757         format_description => "XX:XX:XX:XX:XX:XX",
 758         optional => 1,
 759     },
 760     model => {
 761         type => 'string',
 762         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 763         enum => $nic_model_list,
 764         default_key => 1,
 765     },
 766     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 767     bridge => {
 768         type => 'string',
 769         description => $net_fmt_bridge_descr,
 770         format_description => 'bridge',
 771         optional => 1,
 772     },
 773     queues => {
 774         type => 'integer',
 775         minimum => 0, maximum => 16,
 776         description => 'Number of packet queues to be used on the device.',
 777         optional => 1,
 778     },
 779     rate => {
 780         type => 'number',
 781         minimum => 0,
 782         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 783         optional => 1,
 784     },
 785     tag => {
 786         type => 'integer',
 787         minimum => 1, maximum => 4094,
 788         description => 'VLAN tag to apply to packets on this interface.',
 789         optional => 1,
 790     },
 791     trunks => {
 792         type => 'string',
 793         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 794         description => 'VLAN trunks to pass through this interface.',
 795         format_description => 'vlanid[;vlanid...]',
 796         optional => 1,
 797     },
 798     firewall => {
 799         type => 'boolean',
 800         description => 'Whether this interface should be protected by the firewall.',
 801         optional => 1,
 802     },
 803     link_down => {
 804         type => 'boolean',
 805         description => 'Whether this interface should be disconnected (like pulling the plug).',
 806         optional => 1,
 807     },
 808 };
 809
 810 my $netdesc = {
 811     optional => 1,
 812     type => 'string', format => $net_fmt,
 813     description => "Specify network devices.",
 814 };
 815
 816 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 817
 818 my $ipconfig_fmt = {
 819     ip => {
 820         type => 'string',
 821         format => 'pve-ipv4-config',
 822         format_description => 'IPv4Format/CIDR',
 823         description => 'IPv4 address in CIDR format.',
 824         optional => 1,
 825         default => 'dhcp',
 826     },
 827     gw => {
 828         type => 'string',
 829         format => 'ipv4',
 830         format_description => 'GatewayIPv4',
 831         description => 'Default gateway for IPv4 traffic.',
 832         optional => 1,
 833         requires => 'ip',
 834     },
 835     ip6 => {
 836         type => 'string',
 837         format => 'pve-ipv6-config',
 838         format_description => 'IPv6Format/CIDR',
 839         description => 'IPv6 address in CIDR format.',
 840         optional => 1,
 841         default => 'dhcp',
 842     },
 843     gw6 => {
 844         type => 'string',
 845         format => 'ipv6',
 846         format_description => 'GatewayIPv6',
 847         description => 'Default gateway for IPv6 traffic.',
 848         optional => 1,
 849         requires => 'ip6',
 850     },
 851 };
 852 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 853 my $ipconfigdesc = {
 854     optional => 1,
 855     type => 'string', format => 'pve-qm-ipconfig',
 856     description => <<'EODESCR',
 857 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 858
 859 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 860
 861 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 862 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 863
 864 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 865 EODESCR
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 868
 869 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 870     $confdesc->{"net$i"} = $netdesc;
 871     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 872 }
 873
 874 foreach my $key (keys %$confdesc_cloudinit) {
 875     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 876 }
 877
 878 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 879 sub verify_volume_id_or_qm_path {
 880     my ($volid, $noerr) = @_;
 881
 882     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 883         return $volid;
 884     }
 885
 886     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 887     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 888     if ($@) {
 889         return undef if $noerr;
 890         die $@;
 891     }
 892     return $volid;
 893 }
 894
 895 my $drivename_hash;
 896
 897 my %drivedesc_base = (
 898     volume => { alias => 'file' },
 899     file => {
 900         type => 'string',
 901         format => 'pve-volume-id-or-qm-path',
 902         default_key => 1,
 903         format_description => 'volume',
 904         description => "The drive's backing volume.",
 905     },
 906     media => {
 907         type => 'string',
 908         enum => [qw(cdrom disk)],
 909         description => "The drive's media type.",
 910         default => 'disk',
 911         optional => 1
 912     },
 913     cyls => {
 914         type => 'integer',
 915         description => "Force the drive's physical geometry to have a specific cylinder count.",
 916         optional => 1
 917     },
 918     heads => {
 919         type => 'integer',
 920         description => "Force the drive's physical geometry to have a specific head count.",
 921         optional => 1
 922     },
 923     secs => {
 924         type => 'integer',
 925         description => "Force the drive's physical geometry to have a specific sector count.",
 926         optional => 1
 927     },
 928     trans => {
 929         type => 'string',
 930         enum => [qw(none lba auto)],
 931         description => "Force disk geometry bios translation mode.",
 932         optional => 1,
 933     },
 934     snapshot => {
 935         type => 'boolean',
 936         description => "Controls qemu's snapshot mode feature."
 937             . " If activated, changes made to the disk are temporary and will"
 938             . " be discarded when the VM is shutdown.",
 939         optional => 1,
 940     },
 941     cache => {
 942         type => 'string',
 943         enum => [qw(none writethrough writeback unsafe directsync)],
 944         description => "The drive's cache mode",
 945         optional => 1,
 946     },
 947     format => get_standard_option('pve-qm-image-format'),
 948     size => {
 949         type => 'string',
 950         format => 'disk-size',
 951         format_description => 'DiskSize',
 952         description => "Disk size. This is purely informational and has no effect.",
 953         optional => 1,
 954     },
 955     backup => {
 956         type => 'boolean',
 957         description => "Whether the drive should be included when making backups.",
 958         optional => 1,
 959     },
 960     replicate => {
 961         type => 'boolean',
 962         description => 'Whether the drive should considered for replication jobs.',
 963         optional => 1,
 964         default => 1,
 965     },
 966     rerror => {
 967         type => 'string',
 968         enum => [qw(ignore report stop)],
 969         description => 'Read error action.',
 970         optional => 1,
 971     },
 972     werror => {
 973         type => 'string',
 974         enum => [qw(enospc ignore report stop)],
 975         description => 'Write error action.',
 976         optional => 1,
 977     },
 978     aio => {
 979         type => 'string',
 980         enum => [qw(native threads)],
 981         description => 'AIO type to use.',
 982         optional => 1,
 983     },
 984     discard => {
 985         type => 'string',
 986         enum => [qw(ignore on)],
 987         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 988         optional => 1,
 989     },
 990     detect_zeroes => {
 991         type => 'boolean',
 992         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 993         optional => 1,
 994     },
 995     serial => {
 996         type => 'string',
 997         format => 'urlencoded',
 998         format_description => 'serial',
 999         maxLength => 20*3, # *3 since it's %xx url enoded
1000         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1001         optional => 1,
1002     },
1003     shared => {
1004         type => 'boolean',
1005         description => 'Mark this locally-managed volume as available on all nodes',
1006         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1007         optional => 1,
1008         default => 0,
1009     }
1010 );
1011
1012 my %iothread_fmt = ( iothread => {
1013         type => 'boolean',
1014         description => "Whether to use iothreads for this drive",
1015         optional => 1,
1016 });
1017
1018 my %model_fmt = (
1019     model => {
1020         type => 'string',
1021         format => 'urlencoded',
1022         format_description => 'model',
1023         maxLength => 40*3, # *3 since it's %xx url enoded
1024         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1025         optional => 1,
1026     },
1027 );
1028
1029 my %queues_fmt = (
1030     queues => {
1031         type => 'integer',
1032         description => "Number of queues.",
1033         minimum => 2,
1034         optional => 1
1035     }
1036 );
1037
1038 my %scsiblock_fmt = (
1039     scsiblock => {
1040         type => 'boolean',
1041         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1042         optional => 1,
1043         default => 0,
1044     },
1045 );
1046
1047 my %ssd_fmt = (
1048     ssd => {
1049         type => 'boolean',
1050         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1051         optional => 1,
1052     },
1053 );
1054
1055 my $add_throttle_desc = sub {
1056     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1057     my $d = {
1058         type => $type,
1059         format_description => $unit,
1060         description => "Maximum $what in $longunit.",
1061         optional => 1,
1062     };
1063     $d->{minimum} = $minimum if defined($minimum);
1064     $drivedesc_base{$key} = $d;
1065 };
1066 # throughput: (leaky bucket)
1067 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1068 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1069 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1070 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1071 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1072 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1073 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1074 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1075 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1076
1077 # pools: (pool of IO before throttling starts taking effect)
1078 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1079 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1080 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1081 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1082 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1083 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1084
1085 # burst lengths
1086 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1087 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1088 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1089 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1090 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1091 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1092
1093 # legacy support
1094 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1095 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1096 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1097 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1098
1099 my $ide_fmt = {
1100     %drivedesc_base,
1101     %model_fmt,
1102     %ssd_fmt,
1103 };
1104 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1105
1106 my $idedesc = {
1107     optional => 1,
1108     type => 'string', format => $ide_fmt,
1109     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1112
1113 my $scsi_fmt = {
1114     %drivedesc_base,
1115     %iothread_fmt,
1116     %queues_fmt,
1117     %scsiblock_fmt,
1118     %ssd_fmt,
1119 };
1120 my $scsidesc = {
1121     optional => 1,
1122     type => 'string', format => $scsi_fmt,
1123     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1124 };
1125 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1126
1127 my $sata_fmt = {
1128     %drivedesc_base,
1129     %ssd_fmt,
1130 };
1131 my $satadesc = {
1132     optional => 1,
1133     type => 'string', format => $sata_fmt,
1134     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1135 };
1136 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1137
1138 my $virtio_fmt = {
1139     %drivedesc_base,
1140     %iothread_fmt,
1141 };
1142 my $virtiodesc = {
1143     optional => 1,
1144     type => 'string', format => $virtio_fmt,
1145     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1146 };
1147 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1148
1149 my $alldrive_fmt = {
1150     %drivedesc_base,
1151     %iothread_fmt,
1152     %model_fmt,
1153     %queues_fmt,
1154     %scsiblock_fmt,
1155     %ssd_fmt,
1156 };
1157
1158 my $efidisk_fmt = {
1159     volume => { alias => 'file' },
1160     file => {
1161         type => 'string',
1162         format => 'pve-volume-id-or-qm-path',
1163         default_key => 1,
1164         format_description => 'volume',
1165         description => "The drive's backing volume.",
1166     },
1167     format => get_standard_option('pve-qm-image-format'),
1168     size => {
1169         type => 'string',
1170         format => 'disk-size',
1171         format_description => 'DiskSize',
1172         description => "Disk size. This is purely informational and has no effect.",
1173         optional => 1,
1174     },
1175 };
1176
1177 my $efidisk_desc = {
1178     optional => 1,
1179     type => 'string', format => $efidisk_fmt,
1180     description => "Configure a Disk for storing EFI vars",
1181 };
1182
1183 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1184
1185 my $usb_fmt = {
1186     host => {
1187         default_key => 1,
1188         type => 'string', format => 'pve-qm-usb-device',
1189         format_description => 'HOSTUSBDEVICE|spice',
1190         description => <<EODESCR,
1191 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1192
1193  'bus-port(.port)*' (decimal numbers) or
1194  'vendor_id:product_id' (hexadeciaml numbers) or
1195  'spice'
1196
1197 You can use the 'lsusb -t' command to list existing usb devices.
1198
1199 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1200
1201 The value 'spice' can be used to add a usb redirection devices for spice.
1202 EODESCR
1203     },
1204     usb3 => {
1205         optional => 1,
1206         type => 'boolean',
1207         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1208         default => 0,
1209     },
1210 };
1211
1212 my $usbdesc = {
1213     optional => 1,
1214     type => 'string', format => $usb_fmt,
1215     description => "Configure an USB device (n is 0 to 4).",
1216 };
1217 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1218
1219 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1220 my $hostpci_fmt = {
1221     host => {
1222         default_key => 1,
1223         type => 'string',
1224         pattern => qr/$PCIRE(;$PCIRE)*/,
1225         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1226         description => <<EODESCR,
1227 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1228 of PCI virtual functions of the host. HOSTPCIID syntax is:
1229
1230 'bus:dev.func' (hexadecimal numbers)
1231
1232 You can us the 'lspci' command to list existing PCI devices.
1233 EODESCR
1234     },
1235     rombar => {
1236         type => 'boolean',
1237         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1238         optional => 1,
1239         default => 1,
1240     },
1241     romfile => {
1242         type => 'string',
1243         pattern => '[^,;]+',
1244         format_description => 'string',
1245         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1246         optional => 1,
1247     },
1248     pcie => {
1249         type => 'boolean',
1250         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1251         optional => 1,
1252         default => 0,
1253     },
1254     'x-vga' => {
1255         type => 'boolean',
1256         description =>  "Enable vfio-vga device support.",
1257         optional => 1,
1258         default => 0,
1259     },
1260     'mdev' => {
1261         type => 'string',
1262         format_description => 'string',
1263         pattern => '[^/\.:]+',
1264         optional => 1,
1265         description => <<EODESCR
1266 The type of mediated device to use.
1267 An instance of this type will be created on startup of the VM and
1268 will be cleaned up when the VM stops.
1269 EODESCR
1270     }
1271 };
1272 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1273
1274 my $hostpcidesc = {
1275         optional => 1,
1276         type => 'string', format => 'pve-qm-hostpci',
1277         description => "Map host PCI devices into guest.",
1278         verbose_description =>  <<EODESCR,
1279 Map host PCI devices into guest.
1280
1281 NOTE: This option allows direct access to host hardware. So it is no longer
1282 possible to migrate such machines - use with special care.
1283
1284 CAUTION: Experimental! User reported problems with this option.
1285 EODESCR
1286 };
1287 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1288
1289 my $serialdesc = {
1290         optional => 1,
1291         type => 'string',
1292         pattern => '(/dev/.+|socket)',
1293         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1294         verbose_description =>  <<EODESCR,
1295 Create a serial device inside the VM (n is 0 to 3), and pass through a
1296 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1297 host side (use 'qm terminal' to open a terminal connection).
1298
1299 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1300
1301 CAUTION: Experimental! User reported problems with this option.
1302 EODESCR
1303 };
1304
1305 my $paralleldesc= {
1306         optional => 1,
1307         type => 'string',
1308         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1309         description =>  "Map host parallel devices (n is 0 to 2).",
1310         verbose_description =>  <<EODESCR,
1311 Map host parallel devices (n is 0 to 2).
1312
1313 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1314
1315 CAUTION: Experimental! User reported problems with this option.
1316 EODESCR
1317 };
1318
1319 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1320     $confdesc->{"parallel$i"} = $paralleldesc;
1321 }
1322
1323 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1324     $confdesc->{"serial$i"} = $serialdesc;
1325 }
1326
1327 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1328     $confdesc->{"hostpci$i"} = $hostpcidesc;
1329 }
1330
1331 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1332     $drivename_hash->{"ide$i"} = 1;
1333     $confdesc->{"ide$i"} = $idedesc;
1334 }
1335
1336 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1337     $drivename_hash->{"sata$i"} = 1;
1338     $confdesc->{"sata$i"} = $satadesc;
1339 }
1340
1341 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1342     $drivename_hash->{"scsi$i"} = 1;
1343     $confdesc->{"scsi$i"} = $scsidesc ;
1344 }
1345
1346 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1347     $drivename_hash->{"virtio$i"} = 1;
1348     $confdesc->{"virtio$i"} = $virtiodesc;
1349 }
1350
1351 $drivename_hash->{efidisk0} = 1;
1352 $confdesc->{efidisk0} = $efidisk_desc;
1353
1354 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1355     $confdesc->{"usb$i"} = $usbdesc;
1356 }
1357
1358 my $unuseddesc = {
1359     optional => 1,
1360     type => 'string', format => 'pve-volume-id',
1361     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1362 };
1363
1364 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1365     $confdesc->{"unused$i"} = $unuseddesc;
1366 }
1367
1368 my $kvm_api_version = 0;
1369
1370 sub kvm_version {
1371     return $kvm_api_version if $kvm_api_version;
1372
1373     open my $fh, '<', '/dev/kvm'
1374         or return undef;
1375
1376     # 0xae00 => KVM_GET_API_VERSION
1377     $kvm_api_version = ioctl($fh, 0xae00, 0);
1378
1379     return $kvm_api_version;
1380 }
1381
1382 my $kvm_user_version;
1383
1384 sub kvm_user_version {
1385
1386     return $kvm_user_version if $kvm_user_version;
1387
1388     $kvm_user_version = 'unknown';
1389
1390     my $code = sub {
1391         my $line = shift;
1392         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1393             $kvm_user_version = $2;
1394         }
1395     };
1396
1397     eval { run_command("kvm -version", outfunc => $code); };
1398     warn $@ if $@;
1399
1400     return $kvm_user_version;
1401
1402 }
1403
1404 sub kernel_has_vhost_net {
1405     return -c '/dev/vhost-net';
1406 }
1407
1408 sub valid_drive_names {
1409     # order is important - used to autoselect boot disk
1410     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1411             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1412             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1413             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1414             'efidisk0');
1415 }
1416
1417 sub is_valid_drivename {
1418     my $dev = shift;
1419
1420     return defined($drivename_hash->{$dev});
1421 }
1422
1423 sub option_exists {
1424     my $key = shift;
1425     return defined($confdesc->{$key});
1426 }
1427
1428 sub nic_models {
1429     return $nic_model_list;
1430 }
1431
1432 sub os_list_description {
1433
1434     return {
1435         other => 'Other',
1436         wxp => 'Windows XP',
1437         w2k => 'Windows 2000',
1438         w2k3 =>, 'Windows 2003',
1439         w2k8 => 'Windows 2008',
1440         wvista => 'Windows Vista',
1441         win7 => 'Windows 7',
1442         win8 => 'Windows 8/2012',
1443         win10 => 'Windows 10/2016',
1444         l24 => 'Linux 2.4',
1445         l26 => 'Linux 2.6',
1446     };
1447 }
1448
1449 my $cdrom_path;
1450
1451 sub get_cdrom_path {
1452
1453     return  $cdrom_path if $cdrom_path;
1454
1455     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1456     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1457     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1458 }
1459
1460 sub get_iso_path {
1461     my ($storecfg, $vmid, $cdrom) = @_;
1462
1463     if ($cdrom eq 'cdrom') {
1464         return get_cdrom_path();
1465     } elsif ($cdrom eq 'none') {
1466         return '';
1467     } elsif ($cdrom =~ m|^/|) {
1468         return $cdrom;
1469     } else {
1470         return PVE::Storage::path($storecfg, $cdrom);
1471     }
1472 }
1473
1474 # try to convert old style file names to volume IDs
1475 sub filename_to_volume_id {
1476     my ($vmid, $file, $media) = @_;
1477
1478      if (!($file eq 'none' || $file eq 'cdrom' ||
1479           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1480
1481         return undef if $file =~ m|/|;
1482
1483         if ($media && $media eq 'cdrom') {
1484             $file = "local:iso/$file";
1485         } else {
1486             $file = "local:$vmid/$file";
1487         }
1488     }
1489
1490     return $file;
1491 }
1492
1493 sub verify_media_type {
1494     my ($opt, $vtype, $media) = @_;
1495
1496     return if !$media;
1497
1498     my $etype;
1499     if ($media eq 'disk') {
1500         $etype = 'images';
1501     } elsif ($media eq 'cdrom') {
1502         $etype = 'iso';
1503     } else {
1504         die "internal error";
1505     }
1506
1507     return if ($vtype eq $etype);
1508
1509     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1510 }
1511
1512 sub cleanup_drive_path {
1513     my ($opt, $storecfg, $drive) = @_;
1514
1515     # try to convert filesystem paths to volume IDs
1516
1517     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1518         ($drive->{file} !~ m|^/dev/.+|) &&
1519         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1520         ($drive->{file} !~ m/^\d+$/)) {
1521         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1522         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1523         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1524         verify_media_type($opt, $vtype, $drive->{media});
1525         $drive->{file} = $volid;
1526     }
1527
1528     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1529 }
1530
1531 sub parse_hotplug_features {
1532     my ($data) = @_;
1533
1534     my $res = {};
1535
1536     return $res if $data eq '0';
1537
1538     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1539
1540     foreach my $feature (PVE::Tools::split_list($data)) {
1541         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1542             $res->{$1} = 1;
1543         } else {
1544             die "invalid hotplug feature '$feature'\n";
1545         }
1546     }
1547     return $res;
1548 }
1549
1550 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1551 sub pve_verify_hotplug_features {
1552     my ($value, $noerr) = @_;
1553
1554     return $value if parse_hotplug_features($value);
1555
1556     return undef if $noerr;
1557
1558     die "unable to parse hotplug option\n";
1559 }
1560
1561 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1562 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1563 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1564 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1565 #        [,iothread=on][,serial=serial][,model=model]
1566
1567 sub parse_drive {
1568     my ($key, $data) = @_;
1569
1570     my ($interface, $index);
1571
1572     if ($key =~ m/^([^\d]+)(\d+)$/) {
1573         $interface = $1;
1574         $index = $2;
1575     } else {
1576         return undef;
1577     }
1578
1579     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1580                                      : $confdesc->{$key}->{format};
1581     if (!$desc) {
1582         warn "invalid drive key: $key\n";
1583         return undef;
1584     }
1585     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1586     return undef if !$res;
1587     $res->{interface} = $interface;
1588     $res->{index} = $index;
1589
1590     my $error = 0;
1591     foreach my $opt (qw(bps bps_rd bps_wr)) {
1592         if (my $bps = defined(delete $res->{$opt})) {
1593             if (defined($res->{"m$opt"})) {
1594                 warn "both $opt and m$opt specified\n";
1595                 ++$error;
1596                 next;
1597             }
1598             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1599         }
1600     }
1601
1602     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1603     for my $requirement (
1604         [mbps_max => 'mbps'],
1605         [mbps_rd_max => 'mbps_rd'],
1606         [mbps_wr_max => 'mbps_wr'],
1607         [miops_max => 'miops'],
1608         [miops_rd_max => 'miops_rd'],
1609         [miops_wr_max => 'miops_wr'],
1610         [bps_max_length => 'mbps_max'],
1611         [bps_rd_max_length => 'mbps_rd_max'],
1612         [bps_wr_max_length => 'mbps_wr_max'],
1613         [iops_max_length => 'iops_max'],
1614         [iops_rd_max_length => 'iops_rd_max'],
1615         [iops_wr_max_length => 'iops_wr_max']) {
1616         my ($option, $requires) = @$requirement;
1617         if ($res->{$option} && !$res->{$requires}) {
1618             warn "$option requires $requires\n";
1619             ++$error;
1620         }
1621     }
1622
1623     return undef if $error;
1624
1625     return undef if $res->{mbps_rd} && $res->{mbps};
1626     return undef if $res->{mbps_wr} && $res->{mbps};
1627     return undef if $res->{iops_rd} && $res->{iops};
1628     return undef if $res->{iops_wr} && $res->{iops};
1629
1630     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1631         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1632         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1633         return undef if $res->{interface} eq 'virtio';
1634     }
1635
1636     if (my $size = $res->{size}) {
1637         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1638     }
1639
1640     return $res;
1641 }
1642
1643 sub print_drive {
1644     my ($vmid, $drive) = @_;
1645     my $data = { %$drive };
1646     delete $data->{$_} for qw(index interface);
1647     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1648 }
1649
1650 sub scsi_inquiry {
1651     my($fh, $noerr) = @_;
1652
1653     my $SG_IO = 0x2285;
1654     my $SG_GET_VERSION_NUM = 0x2282;
1655
1656     my $versionbuf = "\x00" x 8;
1657     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1658     if (!$ret) {
1659         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1660         return undef;
1661     }
1662     my $version = unpack("I", $versionbuf);
1663     if ($version < 30000) {
1664         die "scsi generic interface too old\n"  if !$noerr;
1665         return undef;
1666     }
1667
1668     my $buf = "\x00" x 36;
1669     my $sensebuf = "\x00" x 8;
1670     my $cmd = pack("C x3 C x1", 0x12, 36);
1671
1672     # see /usr/include/scsi/sg.h
1673     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1674
1675     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1676                       length($sensebuf), 0, length($buf), $buf,
1677                       $cmd, $sensebuf, 6000);
1678
1679     $ret = ioctl($fh, $SG_IO, $packet);
1680     if (!$ret) {
1681         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1682         return undef;
1683     }
1684
1685     my @res = unpack($sg_io_hdr_t, $packet);
1686     if ($res[17] || $res[18]) {
1687         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1688         return undef;
1689     }
1690
1691     my $res = {};
1692     (my $byte0, my $byte1, $res->{vendor},
1693      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1694
1695     $res->{removable} = $byte1 & 128 ? 1 : 0;
1696     $res->{type} = $byte0 & 31;
1697
1698     return $res;
1699 }
1700
1701 sub path_is_scsi {
1702     my ($path) = @_;
1703
1704     my $fh = IO::File->new("+<$path") || return undef;
1705     my $res = scsi_inquiry($fh, 1);
1706     close($fh);
1707
1708     return $res;
1709 }
1710
1711 sub machine_type_is_q35 {
1712     my ($conf) = @_;
1713
1714     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1715 }
1716
1717 sub print_tabletdevice_full {
1718     my ($conf, $arch) = @_;
1719
1720     my $q35 = machine_type_is_q35($conf);
1721
1722     # we use uhci for old VMs because tablet driver was buggy in older qemu
1723     my $usbbus;
1724     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1725         $usbbus = 'ehci';
1726     } else {
1727         $usbbus = 'uhci';
1728     }
1729
1730     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1731 }
1732
1733 sub print_keyboarddevice_full {
1734     my ($conf, $arch, $machine) = @_;
1735
1736     return undef if $arch ne 'aarch64';
1737
1738     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1739 }
1740
1741 sub print_drivedevice_full {
1742     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1743
1744     my $device = '';
1745     my $maxdev = 0;
1746
1747     if ($drive->{interface} eq 'virtio') {
1748         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1749         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1750         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1751     } elsif ($drive->{interface} eq 'scsi') {
1752
1753         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1754         my $unit = $drive->{index} % $maxdev;
1755         my $devicetype = 'hd';
1756         my $path = '';
1757         if (drive_is_cdrom($drive)) {
1758             $devicetype = 'cd';
1759         } else {
1760             if ($drive->{file} =~ m|^/|) {
1761                 $path = $drive->{file};
1762                 if (my $info = path_is_scsi($path)) {
1763                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1764                         $devicetype = 'block';
1765                     } elsif ($info->{type} == 1) { # tape
1766                         $devicetype = 'generic';
1767                     }
1768                 }
1769             } else {
1770                  $path = PVE::Storage::path($storecfg, $drive->{file});
1771             }
1772
1773             if($path =~ m/^iscsi\:\/\//){
1774                 $devicetype = 'generic';
1775             }
1776         }
1777
1778         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1779             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1780         } else {
1781             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1782         }
1783
1784         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1785             $device .= ",rotation_rate=1";
1786         }
1787
1788     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1789         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1790         my $controller = int($drive->{index} / $maxdev);
1791         my $unit = $drive->{index} % $maxdev;
1792         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1793
1794         $device = "ide-$devicetype";
1795         if ($drive->{interface} eq 'ide') {
1796             $device .= ",bus=ide.$controller,unit=$unit";
1797         } else {
1798             $device .= ",bus=ahci$controller.$unit";
1799         }
1800         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1801
1802         if ($devicetype eq 'hd') {
1803             if (my $model = $drive->{model}) {
1804                 $model = URI::Escape::uri_unescape($model);
1805                 $device .= ",model=$model";
1806             }
1807             if ($drive->{ssd}) {
1808                 $device .= ",rotation_rate=1";
1809             }
1810         }
1811     } elsif ($drive->{interface} eq 'usb') {
1812         die "implement me";
1813         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1814     } else {
1815         die "unsupported interface type";
1816     }
1817
1818     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1819
1820     if (my $serial = $drive->{serial}) {
1821         $serial = URI::Escape::uri_unescape($serial);
1822         $device .= ",serial=$serial";
1823     }
1824
1825
1826     return $device;
1827 }
1828
1829 sub get_initiator_name {
1830     my $initiator;
1831
1832     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1833     while (defined(my $line = <$fh>)) {
1834         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1835         $initiator = $1;
1836         last;
1837     }
1838     $fh->close();
1839
1840     return $initiator;
1841 }
1842
1843 sub print_drive_full {
1844     my ($storecfg, $vmid, $drive) = @_;
1845
1846     my $path;
1847     my $volid = $drive->{file};
1848     my $format;
1849
1850     if (drive_is_cdrom($drive)) {
1851         $path = get_iso_path($storecfg, $vmid, $volid);
1852     } else {
1853         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1854         if ($storeid) {
1855             $path = PVE::Storage::path($storecfg, $volid);
1856             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1857             $format = qemu_img_format($scfg, $volname);
1858         } else {
1859             $path = $volid;
1860             $format = "raw";
1861         }
1862    }
1863
1864     my $opts = '';
1865     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1866     foreach my $o (@qemu_drive_options) {
1867         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1868     }
1869
1870     # snapshot only accepts on|off
1871     if (defined($drive->{snapshot})) {
1872         my $v = $drive->{snapshot} ? 'on' : 'off';
1873         $opts .= ",snapshot=$v";
1874     }
1875
1876     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1877         my ($dir, $qmpname) = @$type;
1878         if (my $v = $drive->{"mbps$dir"}) {
1879             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1880         }
1881         if (my $v = $drive->{"mbps${dir}_max"}) {
1882             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1883         }
1884         if (my $v = $drive->{"bps${dir}_max_length"}) {
1885             $opts .= ",throttling.bps$qmpname-max-length=$v";
1886         }
1887         if (my $v = $drive->{"iops${dir}"}) {
1888             $opts .= ",throttling.iops$qmpname=$v";
1889         }
1890         if (my $v = $drive->{"iops${dir}_max"}) {
1891             $opts .= ",throttling.iops$qmpname-max=$v";
1892         }
1893         if (my $v = $drive->{"iops${dir}_max_length"}) {
1894             $opts .= ",throttling.iops$qmpname-max-length=$v";
1895         }
1896     }
1897
1898     $opts .= ",format=$format" if $format && !$drive->{format};
1899
1900     my $cache_direct = 0;
1901
1902     if (my $cache = $drive->{cache}) {
1903         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1904     } elsif (!drive_is_cdrom($drive)) {
1905         $opts .= ",cache=none";
1906         $cache_direct = 1;
1907     }
1908
1909     # aio native works only with O_DIRECT
1910     if (!$drive->{aio}) {
1911         if($cache_direct) {
1912             $opts .= ",aio=native";
1913         } else {
1914             $opts .= ",aio=threads";
1915         }
1916     }
1917
1918     if (!drive_is_cdrom($drive)) {
1919         my $detectzeroes;
1920         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1921             $detectzeroes = 'off';
1922         } elsif ($drive->{discard}) {
1923             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1924         } else {
1925             # This used to be our default with discard not being specified:
1926             $detectzeroes = 'on';
1927         }
1928         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1929     }
1930
1931     my $pathinfo = $path ? "file=$path," : '';
1932
1933     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1934 }
1935
1936 sub print_netdevice_full {
1937     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1938
1939     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1940
1941     my $device = $net->{model};
1942     if ($net->{model} eq 'virtio') {
1943          $device = 'virtio-net-pci';
1944      };
1945
1946     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1947     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1948     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1949         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1950         my $vectors = $net->{queues} * 2 + 2;
1951         $tmpstr .= ",vectors=$vectors,mq=on";
1952     }
1953     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1954
1955     if ($use_old_bios_files) {
1956         my $romfile;
1957         if ($device eq 'virtio-net-pci') {
1958             $romfile = 'pxe-virtio.rom';
1959         } elsif ($device eq 'e1000') {
1960             $romfile = 'pxe-e1000.rom';
1961         } elsif ($device eq 'ne2k') {
1962             $romfile = 'pxe-ne2k_pci.rom';
1963         } elsif ($device eq 'pcnet') {
1964             $romfile = 'pxe-pcnet.rom';
1965         } elsif ($device eq 'rtl8139') {
1966             $romfile = 'pxe-rtl8139.rom';
1967         }
1968         $tmpstr .= ",romfile=$romfile" if $romfile;
1969     }
1970
1971     return $tmpstr;
1972 }
1973
1974 sub print_netdev_full {
1975     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1976
1977     my $i = '';
1978     if ($netid =~ m/^net(\d+)$/) {
1979         $i = int($1);
1980     }
1981
1982     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1983
1984     my $ifname = "tap${vmid}i$i";
1985
1986     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1987     die "interface name '$ifname' is too long (max 15 character)\n"
1988         if length($ifname) >= 16;
1989
1990     my $vhostparam = '';
1991     if (is_native($arch)) {
1992         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1993     }
1994
1995     my $vmname = $conf->{name} || "vm$vmid";
1996
1997     my $netdev = "";
1998     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1999
2000     if ($net->{bridge}) {
2001         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2002     } else {
2003         $netdev = "type=user,id=$netid,hostname=$vmname";
2004     }
2005
2006     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2007
2008     return $netdev;
2009 }
2010
2011
2012 sub print_cpu_device {
2013     my ($conf, $id) = @_;
2014
2015     my $kvm = $conf->{kvm} // 1;
2016     my $cpu = $kvm ? "kvm64" : "qemu64";
2017     if (my $cputype = $conf->{cpu}) {
2018         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2019             or die "Cannot parse cpu description: $cputype\n";
2020         $cpu = $cpuconf->{cputype};
2021     }
2022
2023     my $cores = $conf->{cores} || 1;
2024
2025     my $current_core = ($id - 1) % $cores;
2026     my $current_socket = int(($id - 1 - $current_core)/$cores);
2027
2028     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2029 }
2030
2031 my $vga_map = {
2032     'cirrus' => 'cirrus-vga',
2033     'std' => 'VGA',
2034     'vmware' => 'vmware-svga',
2035     'virtio' => 'virtio-vga',
2036 };
2037
2038 sub print_vga_device {
2039     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2040
2041     my $type = $vga_map->{$vga->{type}};
2042     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2043         $type = 'virtio-gpu';
2044     }
2045     my $vgamem_mb = $vga->{memory};
2046     if ($qxlnum) {
2047         $type = $id ? 'qxl' : 'qxl-vga';
2048     }
2049     die "no devicetype for $vga->{type}\n" if !$type;
2050
2051     my $memory = "";
2052     if ($vgamem_mb) {
2053         if ($vga->{type} eq 'virtio') {
2054             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2055             $memory = ",max_hostmem=$bytes";
2056         } elsif ($qxlnum) {
2057             # from https://www.spice-space.org/multiple-monitors.html
2058             $memory = ",vgamem_mb=$vga->{memory}";
2059             my $ram = $vgamem_mb * 4;
2060             my $vram = $vgamem_mb * 2;
2061             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2062         } else {
2063             $memory = ",vgamem_mb=$vga->{memory}";
2064         }
2065     } elsif ($qxlnum && $id) {
2066         $memory = ",ram_size=67108864,vram_size=33554432";
2067     }
2068
2069     my $q35 = machine_type_is_q35($conf);
2070     my $vgaid = "vga" . ($id // '');
2071     my $pciaddr;
2072
2073     if ($q35 && $vgaid eq 'vga') {
2074         # the first display uses pcie.0 bus on q35 machines
2075         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2076     } else {
2077         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2078     }
2079
2080     return "$type,id=${vgaid}${memory}${pciaddr}";
2081 }
2082
2083 sub drive_is_cloudinit {
2084     my ($drive) = @_;
2085     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2086 }
2087
2088 sub drive_is_cdrom {
2089     my ($drive, $exclude_cloudinit) = @_;
2090
2091     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2092
2093     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2094
2095 }
2096
2097 sub parse_number_sets {
2098     my ($set) = @_;
2099     my $res = [];
2100     foreach my $part (split(/;/, $set)) {
2101         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2102             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2103             push @$res, [ $1, $2 ];
2104         } else {
2105             die "invalid range: $part\n";
2106         }
2107     }
2108     return $res;
2109 }
2110
2111 sub parse_numa {
2112     my ($data) = @_;
2113
2114     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2115     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2116     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2117     return $res;
2118 }
2119
2120 sub parse_hostpci {
2121     my ($value) = @_;
2122
2123     return undef if !$value;
2124
2125     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2126
2127     my @idlist = split(/;/, $res->{host});
2128     delete $res->{host};
2129     foreach my $id (@idlist) {
2130         if ($id =~ m/\./) { # full id 00:00.1
2131             push @{$res->{pciid}}, {
2132                 id => $id,
2133             };
2134         } else { # partial id 00:00
2135             $res->{pciid} = PVE::SysFSTools::lspci($id);
2136         }
2137     }
2138     return $res;
2139 }
2140
2141 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2142 sub parse_net {
2143     my ($data) = @_;
2144
2145     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2146     if ($@) {
2147         warn $@;
2148         return undef;
2149     }
2150     if (!defined($res->{macaddr})) {
2151         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2152         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2153     }
2154     return $res;
2155 }
2156
2157 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2158 sub parse_ipconfig {
2159     my ($data) = @_;
2160
2161     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2162     if ($@) {
2163         warn $@;
2164         return undef;
2165     }
2166
2167     if ($res->{gw} && !$res->{ip}) {
2168         warn 'gateway specified without specifying an IP address';
2169         return undef;
2170     }
2171     if ($res->{gw6} && !$res->{ip6}) {
2172         warn 'IPv6 gateway specified without specifying an IPv6 address';
2173         return undef;
2174     }
2175     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2176         warn 'gateway specified together with DHCP';
2177         return undef;
2178     }
2179     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2180         # gw6 + auto/dhcp
2181         warn "IPv6 gateway specified together with $res->{ip6} address";
2182         return undef;
2183     }
2184
2185     if (!$res->{ip} && !$res->{ip6}) {
2186         return { ip => 'dhcp', ip6 => 'dhcp' };
2187     }
2188
2189     return $res;
2190 }
2191
2192 sub print_net {
2193     my $net = shift;
2194
2195     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2196 }
2197
2198 sub add_random_macs {
2199     my ($settings) = @_;
2200
2201     foreach my $opt (keys %$settings) {
2202         next if $opt !~ m/^net(\d+)$/;
2203         my $net = parse_net($settings->{$opt});
2204         next if !$net;
2205         $settings->{$opt} = print_net($net);
2206     }
2207 }
2208
2209 sub vm_is_volid_owner {
2210     my ($storecfg, $vmid, $volid) = @_;
2211
2212     if ($volid !~  m|^/|) {
2213         my ($path, $owner);
2214         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2215         if ($owner && ($owner == $vmid)) {
2216             return 1;
2217         }
2218     }
2219
2220     return undef;
2221 }
2222
2223 sub split_flagged_list {
2224     my $text = shift || '';
2225     $text =~ s/[,;]/ /g;
2226     $text =~ s/^\s+//;
2227     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2228 }
2229
2230 sub join_flagged_list {
2231     my ($how, $lst) = @_;
2232     join $how, map { $lst->{$_} . $_ } keys %$lst;
2233 }
2234
2235 sub vmconfig_delete_pending_option {
2236     my ($conf, $key, $force) = @_;
2237
2238     delete $conf->{pending}->{$key};
2239     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2240     $pending_delete_hash->{$key} = $force ? '!' : '';
2241     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2242 }
2243
2244 sub vmconfig_undelete_pending_option {
2245     my ($conf, $key) = @_;
2246
2247     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2248     delete $pending_delete_hash->{$key};
2249
2250     if (%$pending_delete_hash) {
2251         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2252     } else {
2253         delete $conf->{pending}->{delete};
2254     }
2255 }
2256
2257 sub vmconfig_register_unused_drive {
2258     my ($storecfg, $vmid, $conf, $drive) = @_;
2259
2260     if (drive_is_cloudinit($drive)) {
2261         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2262         warn $@ if $@;
2263     } elsif (!drive_is_cdrom($drive)) {
2264         my $volid = $drive->{file};
2265         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2266             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2267         }
2268     }
2269 }
2270
2271 sub vmconfig_cleanup_pending {
2272     my ($conf) = @_;
2273
2274     # remove pending changes when nothing changed
2275     my $changes;
2276     foreach my $opt (keys %{$conf->{pending}}) {
2277         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2278             $changes = 1;
2279             delete $conf->{pending}->{$opt};
2280         }
2281     }
2282
2283     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2284     my $pending_delete_hash = {};
2285     while (my ($opt, $force) = each %$current_delete_hash) {
2286         if (defined($conf->{$opt})) {
2287             $pending_delete_hash->{$opt} = $force;
2288         } else {
2289             $changes = 1;
2290         }
2291     }
2292
2293     if (%$pending_delete_hash) {
2294         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2295     } else {
2296         delete $conf->{pending}->{delete};
2297     }
2298
2299     return $changes;
2300 }
2301
2302 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2303 my $smbios1_fmt = {
2304     uuid => {
2305         type => 'string',
2306         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2307         format_description => 'UUID',
2308         description => "Set SMBIOS1 UUID.",
2309         optional => 1,
2310     },
2311     version => {
2312         type => 'string',
2313         pattern => '\S+',
2314         format_description => 'string',
2315         description => "Set SMBIOS1 version.",
2316         optional => 1,
2317     },
2318     serial => {
2319         type => 'string',
2320         pattern => '\S+',
2321         format_description => 'string',
2322         description => "Set SMBIOS1 serial number.",
2323         optional => 1,
2324     },
2325     manufacturer => {
2326         type => 'string',
2327         pattern => '\S+',
2328         format_description => 'string',
2329         description => "Set SMBIOS1 manufacturer.",
2330         optional => 1,
2331     },
2332     product => {
2333         type => 'string',
2334         pattern => '\S+',
2335         format_description => 'string',
2336         description => "Set SMBIOS1 product ID.",
2337         optional => 1,
2338     },
2339     sku => {
2340         type => 'string',
2341         pattern => '\S+',
2342         format_description => 'string',
2343         description => "Set SMBIOS1 SKU string.",
2344         optional => 1,
2345     },
2346     family => {
2347         type => 'string',
2348         pattern => '\S+',
2349         format_description => 'string',
2350         description => "Set SMBIOS1 family string.",
2351         optional => 1,
2352     },
2353 };
2354
2355 sub parse_smbios1 {
2356     my ($data) = @_;
2357
2358     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2359     warn $@ if $@;
2360     return $res;
2361 }
2362
2363 sub print_smbios1 {
2364     my ($smbios1) = @_;
2365     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2366 }
2367
2368 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2369
2370 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2371 sub verify_bootdisk {
2372     my ($value, $noerr) = @_;
2373
2374     return $value if is_valid_drivename($value);
2375
2376     return undef if $noerr;
2377
2378     die "invalid boot disk '$value'\n";
2379 }
2380
2381 sub parse_watchdog {
2382     my ($value) = @_;
2383
2384     return undef if !$value;
2385
2386     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2387     warn $@ if $@;
2388     return $res;
2389 }
2390
2391 sub parse_guest_agent {
2392     my ($value) = @_;
2393
2394     return {} if !defined($value->{agent});
2395
2396     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2397     warn $@ if $@;
2398
2399     # if the agent is disabled ignore the other potentially set properties
2400     return {} if !$res->{enabled};
2401     return $res;
2402 }
2403
2404 sub parse_vga {
2405     my ($value) = @_;
2406
2407     return {} if !$value;
2408     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2409     warn $@ if $@;
2410     return $res;
2411 }
2412
2413 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2414 sub verify_usb_device {
2415     my ($value, $noerr) = @_;
2416
2417     return $value if parse_usb_device($value);
2418
2419     return undef if $noerr;
2420
2421     die "unable to parse usb device\n";
2422 }
2423
2424 # add JSON properties for create and set function
2425 sub json_config_properties {
2426     my $prop = shift;
2427
2428     foreach my $opt (keys %$confdesc) {
2429         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2430         $prop->{$opt} = $confdesc->{$opt};
2431     }
2432
2433     return $prop;
2434 }
2435
2436 # return copy of $confdesc_cloudinit to generate documentation
2437 sub cloudinit_config_properties {
2438
2439     return dclone($confdesc_cloudinit);
2440 }
2441
2442 sub check_type {
2443     my ($key, $value) = @_;
2444
2445     die "unknown setting '$key'\n" if !$confdesc->{$key};
2446
2447     my $type = $confdesc->{$key}->{type};
2448
2449     if (!defined($value)) {
2450         die "got undefined value\n";
2451     }
2452
2453     if ($value =~ m/[\n\r]/) {
2454         die "property contains a line feed\n";
2455     }
2456
2457     if ($type eq 'boolean') {
2458         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2459         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2460         die "type check ('boolean') failed - got '$value'\n";
2461     } elsif ($type eq 'integer') {
2462         return int($1) if $value =~ m/^(\d+)$/;
2463         die "type check ('integer') failed - got '$value'\n";
2464     } elsif ($type eq 'number') {
2465         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2466         die "type check ('number') failed - got '$value'\n";
2467     } elsif ($type eq 'string') {
2468         if (my $fmt = $confdesc->{$key}->{format}) {
2469             PVE::JSONSchema::check_format($fmt, $value);
2470             return $value;
2471         }
2472         $value =~ s/^\"(.*)\"$/$1/;
2473         return $value;
2474     } else {
2475         die "internal error"
2476     }
2477 }
2478
2479 sub touch_config {
2480     my ($vmid) = @_;
2481
2482     my $conf = PVE::QemuConfig->config_file($vmid);
2483     utime undef, undef, $conf;
2484 }
2485
2486 sub destroy_vm {
2487     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2488
2489     my $conffile = PVE::QemuConfig->config_file($vmid);
2490
2491     my $conf = PVE::QemuConfig->load_config($vmid);
2492
2493     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2494
2495     if ($conf->{template}) {
2496         # check if any base image is still used by a linked clone
2497         foreach_drive($conf, sub {
2498                 my ($ds, $drive) = @_;
2499
2500                 return if drive_is_cdrom($drive);
2501
2502                 my $volid = $drive->{file};
2503
2504                 return if !$volid || $volid =~ m|^/|;
2505
2506                 die "base volume '$volid' is still in use by linked cloned\n"
2507                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2508
2509         });
2510     }
2511
2512     # only remove disks owned by this VM
2513     foreach_drive($conf, sub {
2514         my ($ds, $drive) = @_;
2515
2516         return if drive_is_cdrom($drive, 1);
2517
2518         my $volid = $drive->{file};
2519
2520         return if !$volid || $volid =~ m|^/|;
2521
2522         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2523         return if !$path || !$owner || ($owner != $vmid);
2524
2525         eval {
2526             PVE::Storage::vdisk_free($storecfg, $volid);
2527         };
2528         warn "Could not remove disk '$volid', check manually: $@" if $@;
2529
2530     });
2531
2532     if ($keep_empty_config) {
2533         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2534     } else {
2535         unlink $conffile;
2536     }
2537
2538     # also remove unused disk
2539     eval {
2540         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2541
2542         eval {
2543             PVE::Storage::foreach_volid($dl, sub {
2544                 my ($volid, $sid, $volname, $d) = @_;
2545                 PVE::Storage::vdisk_free($storecfg, $volid);
2546             });
2547         };
2548         warn $@ if $@;
2549
2550     };
2551     warn $@ if $@;
2552 }
2553
2554 sub parse_vm_config {
2555     my ($filename, $raw) = @_;
2556
2557     return undef if !defined($raw);
2558
2559     my $res = {
2560         digest => Digest::SHA::sha1_hex($raw),
2561         snapshots => {},
2562         pending => {},
2563     };
2564
2565     $filename =~ m|/qemu-server/(\d+)\.conf$|
2566         || die "got strange filename '$filename'";
2567
2568     my $vmid = $1;
2569
2570     my $conf = $res;
2571     my $descr;
2572     my $section = '';
2573
2574     my @lines = split(/\n/, $raw);
2575     foreach my $line (@lines) {
2576         next if $line =~ m/^\s*$/;
2577
2578         if ($line =~ m/^\[PENDING\]\s*$/i) {
2579             $section = 'pending';
2580             if (defined($descr)) {
2581                 $descr =~ s/\s+$//;
2582                 $conf->{description} = $descr;
2583             }
2584             $descr = undef;
2585             $conf = $res->{$section} = {};
2586             next;
2587
2588         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2589             $section = $1;
2590             if (defined($descr)) {
2591                 $descr =~ s/\s+$//;
2592                 $conf->{description} = $descr;
2593             }
2594             $descr = undef;
2595             $conf = $res->{snapshots}->{$section} = {};
2596             next;
2597         }
2598
2599         if ($line =~ m/^\#(.*)\s*$/) {
2600             $descr = '' if !defined($descr);
2601             $descr .= PVE::Tools::decode_text($1) . "\n";
2602             next;
2603         }
2604
2605         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2606             $descr = '' if !defined($descr);
2607             $descr .= PVE::Tools::decode_text($2);
2608         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2609             $conf->{snapstate} = $1;
2610         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2611             my $key = $1;
2612             my $value = $2;
2613             $conf->{$key} = $value;
2614         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2615             my $value = $1;
2616             if ($section eq 'pending') {
2617                 $conf->{delete} = $value; # we parse this later
2618             } else {
2619                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2620             }
2621         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2622             my $key = $1;
2623             my $value = $2;
2624             eval { $value = check_type($key, $value); };
2625             if ($@) {
2626                 warn "vm $vmid - unable to parse value of '$key' - $@";
2627             } else {
2628                 $key = 'ide2' if $key eq 'cdrom';
2629                 my $fmt = $confdesc->{$key}->{format};
2630                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2631                     my $v = parse_drive($key, $value);
2632                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2633                         $v->{file} = $volid;
2634                         $value = print_drive($vmid, $v);
2635                     } else {
2636                         warn "vm $vmid - unable to parse value of '$key'\n";
2637                         next;
2638                     }
2639                 }
2640
2641                 $conf->{$key} = $value;
2642             }
2643         }
2644     }
2645
2646     if (defined($descr)) {
2647         $descr =~ s/\s+$//;
2648         $conf->{description} = $descr;
2649     }
2650     delete $res->{snapstate}; # just to be sure
2651
2652     return $res;
2653 }
2654
2655 sub write_vm_config {
2656     my ($filename, $conf) = @_;
2657
2658     delete $conf->{snapstate}; # just to be sure
2659
2660     if ($conf->{cdrom}) {
2661         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2662         $conf->{ide2} = $conf->{cdrom};
2663         delete $conf->{cdrom};
2664     }
2665
2666     # we do not use 'smp' any longer
2667     if ($conf->{sockets}) {
2668         delete $conf->{smp};
2669     } elsif ($conf->{smp}) {
2670         $conf->{sockets} = $conf->{smp};
2671         delete $conf->{cores};
2672         delete $conf->{smp};
2673     }
2674
2675     my $used_volids = {};
2676
2677     my $cleanup_config = sub {
2678         my ($cref, $pending, $snapname) = @_;
2679
2680         foreach my $key (keys %$cref) {
2681             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2682                 $key eq 'snapstate' || $key eq 'pending';
2683             my $value = $cref->{$key};
2684             if ($key eq 'delete') {
2685                 die "propertry 'delete' is only allowed in [PENDING]\n"
2686                     if !$pending;
2687                 # fixme: check syntax?
2688                 next;
2689             }
2690             eval { $value = check_type($key, $value); };
2691             die "unable to parse value of '$key' - $@" if $@;
2692
2693             $cref->{$key} = $value;
2694
2695             if (!$snapname && is_valid_drivename($key)) {
2696                 my $drive = parse_drive($key, $value);
2697                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2698             }
2699         }
2700     };
2701
2702     &$cleanup_config($conf);
2703
2704     &$cleanup_config($conf->{pending}, 1);
2705
2706     foreach my $snapname (keys %{$conf->{snapshots}}) {
2707         die "internal error" if $snapname eq 'pending';
2708         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2709     }
2710
2711     # remove 'unusedX' settings if we re-add a volume
2712     foreach my $key (keys %$conf) {
2713         my $value = $conf->{$key};
2714         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2715             delete $conf->{$key};
2716         }
2717     }
2718
2719     my $generate_raw_config = sub {
2720         my ($conf, $pending) = @_;
2721
2722         my $raw = '';
2723
2724         # add description as comment to top of file
2725         if (defined(my $descr = $conf->{description})) {
2726             if ($descr) {
2727                 foreach my $cl (split(/\n/, $descr)) {
2728                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2729                 }
2730             } else {
2731                 $raw .= "#\n" if $pending;
2732             }
2733         }
2734
2735         foreach my $key (sort keys %$conf) {
2736             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2737             $raw .= "$key: $conf->{$key}\n";
2738         }
2739         return $raw;
2740     };
2741
2742     my $raw = &$generate_raw_config($conf);
2743
2744     if (scalar(keys %{$conf->{pending}})){
2745         $raw .= "\n[PENDING]\n";
2746         $raw .= &$generate_raw_config($conf->{pending}, 1);
2747     }
2748
2749     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2750         $raw .= "\n[$snapname]\n";
2751         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2752     }
2753
2754     return $raw;
2755 }
2756
2757 sub load_defaults {
2758
2759     my $res = {};
2760
2761     # we use static defaults from our JSON schema configuration
2762     foreach my $key (keys %$confdesc) {
2763         if (defined(my $default = $confdesc->{$key}->{default})) {
2764             $res->{$key} = $default;
2765         }
2766     }
2767
2768     return $res;
2769 }
2770
2771 sub config_list {
2772     my $vmlist = PVE::Cluster::get_vmlist();
2773     my $res = {};
2774     return $res if !$vmlist || !$vmlist->{ids};
2775     my $ids = $vmlist->{ids};
2776
2777     foreach my $vmid (keys %$ids) {
2778         my $d = $ids->{$vmid};
2779         next if !$d->{node} || $d->{node} ne $nodename;
2780         next if !$d->{type} || $d->{type} ne 'qemu';
2781         $res->{$vmid}->{exists} = 1;
2782     }
2783     return $res;
2784 }
2785
2786 # test if VM uses local resources (to prevent migration)
2787 sub check_local_resources {
2788     my ($conf, $noerr) = @_;
2789
2790     my $loc_res = 0;
2791
2792     $loc_res = 1 if $conf->{hostusb}; # old syntax
2793     $loc_res = 1 if $conf->{hostpci}; # old syntax
2794
2795     foreach my $k (keys %$conf) {
2796         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2797         # sockets are safe: they will recreated be on the target side post-migrate
2798         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2799         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2800     }
2801
2802     die "VM uses local resources\n" if $loc_res && !$noerr;
2803
2804     return $loc_res;
2805 }
2806
2807 # check if used storages are available on all nodes (use by migrate)
2808 sub check_storage_availability {
2809     my ($storecfg, $conf, $node) = @_;
2810
2811     foreach_drive($conf, sub {
2812         my ($ds, $drive) = @_;
2813
2814         my $volid = $drive->{file};
2815         return if !$volid;
2816
2817         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2818         return if !$sid;
2819
2820         # check if storage is available on both nodes
2821         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2822         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2823    });
2824 }
2825
2826 # list nodes where all VM images are available (used by has_feature API)
2827 sub shared_nodes {
2828     my ($conf, $storecfg) = @_;
2829
2830     my $nodelist = PVE::Cluster::get_nodelist();
2831     my $nodehash = { map { $_ => 1 } @$nodelist };
2832     my $nodename = PVE::INotify::nodename();
2833
2834     foreach_drive($conf, sub {
2835         my ($ds, $drive) = @_;
2836
2837         my $volid = $drive->{file};
2838         return if !$volid;
2839
2840         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2841         if ($storeid) {
2842             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2843             if ($scfg->{disable}) {
2844                 $nodehash = {};
2845             } elsif (my $avail = $scfg->{nodes}) {
2846                 foreach my $node (keys %$nodehash) {
2847                     delete $nodehash->{$node} if !$avail->{$node};
2848                 }
2849             } elsif (!$scfg->{shared}) {
2850                 foreach my $node (keys %$nodehash) {
2851                     delete $nodehash->{$node} if $node ne $nodename
2852                 }
2853             }
2854         }
2855     });
2856
2857     return $nodehash
2858 }
2859
2860 sub check_cmdline {
2861     my ($pidfile, $pid) = @_;
2862
2863     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2864     if (defined($fh)) {
2865         my $line = <$fh>;
2866         $fh->close;
2867         return undef if !$line;
2868         my @param = split(/\0/, $line);
2869
2870         my $cmd = $param[0];
2871         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2872
2873         for (my $i = 0; $i < scalar (@param); $i++) {
2874             my $p = $param[$i];
2875             next if !$p;
2876             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2877                 my $p = $param[$i+1];
2878                 return 1 if $p && ($p eq $pidfile);
2879                 return undef;
2880             }
2881         }
2882     }
2883     return undef;
2884 }
2885
2886 sub check_running {
2887     my ($vmid, $nocheck, $node) = @_;
2888
2889     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2890
2891     die "unable to find configuration file for VM $vmid - no such machine\n"
2892         if !$nocheck && ! -f $filename;
2893
2894     my $pidfile = pidfile_name($vmid);
2895
2896     if (my $fd = IO::File->new("<$pidfile")) {
2897         my $st = stat($fd);
2898         my $line = <$fd>;
2899         close($fd);
2900
2901         my $mtime = $st->mtime;
2902         if ($mtime > time()) {
2903             warn "file '$filename' modified in future\n";
2904         }
2905
2906         if ($line =~ m/^(\d+)$/) {
2907             my $pid = $1;
2908             if (check_cmdline($pidfile, $pid)) {
2909                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2910                     return $pid;
2911                 }
2912             }
2913         }
2914     }
2915
2916     return undef;
2917 }
2918
2919 sub vzlist {
2920
2921     my $vzlist = config_list();
2922
2923     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2924
2925     while (defined(my $de = $fd->read)) {
2926         next if $de !~ m/^(\d+)\.pid$/;
2927         my $vmid = $1;
2928         next if !defined($vzlist->{$vmid});
2929         if (my $pid = check_running($vmid)) {
2930             $vzlist->{$vmid}->{pid} = $pid;
2931         }
2932     }
2933
2934     return $vzlist;
2935 }
2936
2937 sub disksize {
2938     my ($storecfg, $conf) = @_;
2939
2940     my $bootdisk = $conf->{bootdisk};
2941     return undef if !$bootdisk;
2942     return undef if !is_valid_drivename($bootdisk);
2943
2944     return undef if !$conf->{$bootdisk};
2945
2946     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2947     return undef if !defined($drive);
2948
2949     return undef if drive_is_cdrom($drive);
2950
2951     my $volid = $drive->{file};
2952     return undef if !$volid;
2953
2954     return $drive->{size};
2955 }
2956
2957 our $vmstatus_return_properties = {
2958     vmid => get_standard_option('pve-vmid'),
2959     status => {
2960         description => "Qemu process status.",
2961         type => 'string',
2962         enum => ['stopped', 'running'],
2963     },
2964     maxmem => {
2965         description => "Maximum memory in bytes.",
2966         type => 'integer',
2967         optional => 1,
2968         renderer => 'bytes',
2969     },
2970     maxdisk => {
2971         description => "Root disk size in bytes.",
2972         type => 'integer',
2973         optional => 1,
2974         renderer => 'bytes',
2975     },
2976     name => {
2977         description => "VM name.",
2978         type => 'string',
2979         optional => 1,
2980     },
2981     qmpstatus => {
2982         description => "Qemu QMP agent status.",
2983         type => 'string',
2984         optional => 1,
2985     },
2986     pid => {
2987         description => "PID of running qemu process.",
2988         type => 'integer',
2989         optional => 1,
2990     },
2991     uptime => {
2992         description => "Uptime.",
2993         type => 'integer',
2994         optional => 1,
2995         renderer => 'duration',
2996     },
2997     cpus => {
2998         description => "Maximum usable CPUs.",
2999         type => 'number',
3000         optional => 1,
3001     },
3002 };
3003
3004 my $last_proc_pid_stat;
3005
3006 # get VM status information
3007 # This must be fast and should not block ($full == false)
3008 # We only query KVM using QMP if $full == true (this can be slow)
3009 sub vmstatus {
3010     my ($opt_vmid, $full) = @_;
3011
3012     my $res = {};
3013
3014     my $storecfg = PVE::Storage::config();
3015
3016     my $list = vzlist();
3017     my $defaults = load_defaults();
3018
3019     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3020
3021     my $cpucount = $cpuinfo->{cpus} || 1;
3022
3023     foreach my $vmid (keys %$list) {
3024         next if $opt_vmid && ($vmid ne $opt_vmid);
3025
3026         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3027         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3028
3029         my $d = { vmid => $vmid };
3030         $d->{pid} = $list->{$vmid}->{pid};
3031
3032         # fixme: better status?
3033         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3034
3035         my $size = disksize($storecfg, $conf);
3036         if (defined($size)) {
3037             $d->{disk} = 0; # no info available
3038             $d->{maxdisk} = $size;
3039         } else {
3040             $d->{disk} = 0;
3041             $d->{maxdisk} = 0;
3042         }
3043
3044         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3045             * ($conf->{cores} || $defaults->{cores});
3046         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3047         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3048
3049         $d->{name} = $conf->{name} || "VM $vmid";
3050         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3051             : $defaults->{memory}*(1024*1024);
3052
3053         if ($conf->{balloon}) {
3054             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3055             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3056                 : $defaults->{shares};
3057         }
3058
3059         $d->{uptime} = 0;
3060         $d->{cpu} = 0;
3061         $d->{mem} = 0;
3062
3063         $d->{netout} = 0;
3064         $d->{netin} = 0;
3065
3066         $d->{diskread} = 0;
3067         $d->{diskwrite} = 0;
3068
3069         $d->{template} = PVE::QemuConfig->is_template($conf);
3070
3071         $d->{serial} = 1 if conf_has_serial($conf);
3072
3073         $res->{$vmid} = $d;
3074     }
3075
3076     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3077     foreach my $dev (keys %$netdev) {
3078         next if $dev !~ m/^tap([1-9]\d*)i/;
3079         my $vmid = $1;
3080         my $d = $res->{$vmid};
3081         next if !$d;
3082
3083         $d->{netout} += $netdev->{$dev}->{receive};
3084         $d->{netin} += $netdev->{$dev}->{transmit};
3085
3086         if ($full) {
3087             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3088             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3089         }
3090
3091     }
3092
3093     my $ctime = gettimeofday;
3094
3095     foreach my $vmid (keys %$list) {
3096
3097         my $d = $res->{$vmid};
3098         my $pid = $d->{pid};
3099         next if !$pid;
3100
3101         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3102         next if !$pstat; # not running
3103
3104         my $used = $pstat->{utime} + $pstat->{stime};
3105
3106         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3107
3108         if ($pstat->{vsize}) {
3109             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3110         }
3111
3112         my $old = $last_proc_pid_stat->{$pid};
3113         if (!$old) {
3114             $last_proc_pid_stat->{$pid} = {
3115                 time => $ctime,
3116                 used => $used,
3117                 cpu => 0,
3118             };
3119             next;
3120         }
3121
3122         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3123
3124         if ($dtime > 1000) {
3125             my $dutime = $used -  $old->{used};
3126
3127             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3128             $last_proc_pid_stat->{$pid} = {
3129                 time => $ctime,
3130                 used => $used,
3131                 cpu => $d->{cpu},
3132             };
3133         } else {
3134             $d->{cpu} = $old->{cpu};
3135         }
3136     }
3137
3138     return $res if !$full;
3139
3140     my $qmpclient = PVE::QMPClient->new();
3141
3142     my $ballooncb = sub {
3143         my ($vmid, $resp) = @_;
3144
3145         my $info = $resp->{'return'};
3146         return if !$info->{max_mem};
3147
3148         my $d = $res->{$vmid};
3149
3150         # use memory assigned to VM
3151         $d->{maxmem} = $info->{max_mem};
3152         $d->{balloon} = $info->{actual};
3153
3154         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3155             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3156             $d->{freemem} = $info->{free_mem};
3157         }
3158
3159         $d->{ballooninfo} = $info;
3160     };
3161
3162     my $blockstatscb = sub {
3163         my ($vmid, $resp) = @_;
3164         my $data = $resp->{'return'} || [];
3165         my $totalrdbytes = 0;
3166         my $totalwrbytes = 0;
3167
3168         for my $blockstat (@$data) {
3169             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3170             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3171
3172             $blockstat->{device} =~ s/drive-//;
3173             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3174         }
3175         $res->{$vmid}->{diskread} = $totalrdbytes;
3176         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3177     };
3178
3179     my $statuscb = sub {
3180         my ($vmid, $resp) = @_;
3181
3182         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3183         # this fails if ballon driver is not loaded, so this must be
3184         # the last commnand (following command are aborted if this fails).
3185         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3186
3187         my $status = 'unknown';
3188         if (!defined($status = $resp->{'return'}->{status})) {
3189             warn "unable to get VM status\n";
3190             return;
3191         }
3192
3193         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3194     };
3195
3196     foreach my $vmid (keys %$list) {
3197         next if $opt_vmid && ($vmid ne $opt_vmid);
3198         next if !$res->{$vmid}->{pid}; # not running
3199         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3200     }
3201
3202     $qmpclient->queue_execute(undef, 2);
3203
3204     foreach my $vmid (keys %$list) {
3205         next if $opt_vmid && ($vmid ne $opt_vmid);
3206         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3207     }
3208
3209     return $res;
3210 }
3211
3212 sub foreach_drive {
3213     my ($conf, $func, @param) = @_;
3214
3215     foreach my $ds (valid_drive_names()) {
3216         next if !defined($conf->{$ds});
3217
3218         my $drive = parse_drive($ds, $conf->{$ds});
3219         next if !$drive;
3220
3221         &$func($ds, $drive, @param);
3222     }
3223 }
3224
3225 sub foreach_volid {
3226     my ($conf, $func, @param) = @_;
3227
3228     my $volhash = {};
3229
3230     my $test_volid = sub {
3231         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3232
3233         return if !$volid;
3234
3235         $volhash->{$volid}->{cdrom} //= 1;
3236         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3237
3238         $volhash->{$volid}->{replicate} //= 0;
3239         $volhash->{$volid}->{replicate} = 1 if $replicate;
3240
3241         $volhash->{$volid}->{shared} //= 0;
3242         $volhash->{$volid}->{shared} = 1 if $shared;
3243
3244         $volhash->{$volid}->{referenced_in_config} //= 0;
3245         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3246
3247         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3248             if defined($snapname);
3249     };
3250
3251     foreach_drive($conf, sub {
3252         my ($ds, $drive) = @_;
3253         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3254     });
3255
3256     foreach my $snapname (keys %{$conf->{snapshots}}) {
3257         my $snap = $conf->{snapshots}->{$snapname};
3258         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3259         foreach_drive($snap, sub {
3260             my ($ds, $drive) = @_;
3261             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3262         });
3263     }
3264
3265     foreach my $volid (keys %$volhash) {
3266         &$func($volid, $volhash->{$volid}, @param);
3267     }
3268 }
3269
3270 sub conf_has_serial {
3271     my ($conf) = @_;
3272
3273     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3274         if ($conf->{"serial$i"}) {
3275             return 1;
3276         }
3277     }
3278
3279     return 0;
3280 }
3281
3282 sub vga_conf_has_spice {
3283     my ($vga) = @_;
3284
3285     my $vgaconf = parse_vga($vga);
3286     my $vgatype = $vgaconf->{type};
3287     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3288
3289     return $1 || 1;
3290 }
3291
3292 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3293 sub get_host_arch() {
3294     $host_arch = (POSIX::uname())[4] if !$host_arch;
3295     return $host_arch;
3296 }
3297
3298 sub is_native($) {
3299     my ($arch) = @_;
3300     return get_host_arch() eq $arch;
3301 }
3302
3303 my $default_machines = {
3304     x86_64 => 'pc',
3305     aarch64 => 'virt',
3306 };
3307
3308 sub get_basic_machine_info {
3309     my ($conf, $forcemachine) = @_;
3310
3311     my $arch = $conf->{arch} // get_host_arch();
3312     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3313     return ($arch, $machine);
3314 }
3315
3316 sub get_ovmf_files($) {
3317     my ($arch) = @_;
3318
3319     my $ovmf = $OVMF->{$arch}
3320         or die "no OVMF images known for architecture '$arch'\n";
3321
3322     return @$ovmf;
3323 }
3324
3325 my $Arch2Qemu = {
3326     aarch64 => '/usr/bin/qemu-system-aarch64',
3327     x86_64 => '/usr/bin/qemu-system-x86_64',
3328 };
3329 sub get_command_for_arch($) {
3330     my ($arch) = @_;
3331     return '/usr/bin/kvm' if is_native($arch);
3332
3333     my $cmd = $Arch2Qemu->{$arch}
3334         or die "don't know how to emulate architecture '$arch'\n";
3335     return $cmd;
3336 }
3337
3338 sub get_cpu_options {
3339     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3340
3341     my $cpuFlags = [];
3342     my $ostype = $conf->{ostype};
3343
3344     my $cpu = $kvm ? "kvm64" : "qemu64";
3345     if ($arch eq 'aarch64') {
3346         $cpu = 'cortex-a57';
3347     }
3348     my $hv_vendor_id;
3349     if (my $cputype = $conf->{cpu}) {
3350         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3351             or die "Cannot parse cpu description: $cputype\n";
3352         $cpu = $cpuconf->{cputype};
3353         $kvm_off = 1 if $cpuconf->{hidden};
3354         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3355
3356         if (defined(my $flags = $cpuconf->{flags})) {
3357             push @$cpuFlags, split(";", $flags);
3358         }
3359     }
3360
3361     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3362
3363     push @$cpuFlags , '-x2apic'
3364         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3365
3366     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3367
3368     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3369
3370     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3371
3372         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3373         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3374     }
3375
3376     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3377
3378     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3379
3380     push @$cpuFlags, 'kvm=off' if $kvm_off;
3381
3382     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3383         push @$cpuFlags, "vendor=${cpu_vendor}"
3384             if $cpu_vendor ne 'default';
3385     } elsif ($arch ne 'aarch64') {
3386         die "internal error"; # should not happen
3387     }
3388
3389     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3390
3391     return ('-cpu', $cpu);
3392 }
3393
3394 sub config_to_command {
3395     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3396
3397     my $cmd = [];
3398     my $globalFlags = [];
3399     my $machineFlags = [];
3400     my $rtcFlags = [];
3401     my $devices = [];
3402     my $pciaddr = '';
3403     my $bridges = {};
3404     my $kvmver = kvm_user_version();
3405     my $vernum = 0; # unknown
3406     my $ostype = $conf->{ostype};
3407     my $winversion = windows_version($ostype);
3408     my $kvm = $conf->{kvm};
3409
3410     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3411     $kvm //= 1 if is_native($arch);
3412
3413     if ($kvm) {
3414         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3415             if !defined kvm_version();
3416     }
3417
3418     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3419         $vernum = $1*1000000+$2*1000;
3420     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3421         $vernum = $1*1000000+$2*1000+$3;
3422     }
3423
3424     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3425
3426     my $have_ovz = -f '/proc/vz/vestat';
3427
3428     my $q35 = machine_type_is_q35($conf);
3429     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3430     my $use_old_bios_files = undef;
3431     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3432
3433     my $cpuunits = defined($conf->{cpuunits}) ?
3434             $conf->{cpuunits} : $defaults->{cpuunits};
3435
3436     push @$cmd, get_command_for_arch($arch);
3437
3438     push @$cmd, '-id', $vmid;
3439
3440     my $vmname = $conf->{name} || "vm$vmid";
3441
3442     push @$cmd, '-name', $vmname;
3443
3444     my $use_virtio = 0;
3445
3446     my $qmpsocket = qmp_socket($vmid);
3447     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3448     push @$cmd, '-mon', "chardev=qmp,mode=control";
3449
3450     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3451         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3452         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3453     }
3454
3455     push @$cmd, '-pidfile' , pidfile_name($vmid);
3456
3457     push @$cmd, '-daemonize';
3458
3459     if ($conf->{smbios1}) {
3460         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3461     }
3462
3463     if ($conf->{vmgenid}) {
3464         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3465     }
3466
3467     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3468     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3469         die "uefi base image not found\n" if ! -f $ovmf_code;
3470
3471         my $path;
3472         my $format;
3473         if (my $efidisk = $conf->{efidisk0}) {
3474             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3475             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3476             $format = $d->{format};
3477             if ($storeid) {
3478                 $path = PVE::Storage::path($storecfg, $d->{file});
3479                 if (!defined($format)) {
3480                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3481                     $format = qemu_img_format($scfg, $volname);
3482                 }
3483             } else {
3484                 $path = $d->{file};
3485                 die "efidisk format must be specified\n"
3486                     if !defined($format);
3487             }
3488         } else {
3489             warn "no efidisk configured! Using temporary efivars disk.\n";
3490             $path = "/tmp/$vmid-ovmf.fd";
3491             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3492             $format = 'raw';
3493         }
3494
3495         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3496         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3497     }
3498
3499
3500     # add usb controllers
3501     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3502     push @$devices, @usbcontrollers if @usbcontrollers;
3503     my $vga = parse_vga($conf->{vga});
3504
3505     my $qxlnum = vga_conf_has_spice($conf->{vga});
3506     $vga->{type} = 'qxl' if $qxlnum;
3507
3508     if (!$vga->{type}) {
3509         if ($arch eq 'aarch64') {
3510             $vga->{type} = 'virtio';
3511         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3512             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3513         } else {
3514             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3515         }
3516     }
3517
3518     # enable absolute mouse coordinates (needed by vnc)
3519     my $tablet;
3520     if (defined($conf->{tablet})) {
3521         $tablet = $conf->{tablet};
3522     } else {
3523         $tablet = $defaults->{tablet};
3524         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3525         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3526     }
3527
3528     if ($tablet) {
3529         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3530         my $kbd = print_keyboarddevice_full($conf, $arch);
3531         push @$devices, '-device', $kbd if defined($kbd);
3532     }
3533
3534     my $kvm_off = 0;
3535     my $gpu_passthrough;
3536
3537     # host pci devices
3538     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3539         my $d = parse_hostpci($conf->{"hostpci$i"});
3540         next if !$d;
3541
3542         my $pcie = $d->{pcie};
3543         if($pcie){
3544             die "q35 machine model is not enabled" if !$q35;
3545             # win7 wants to have the pcie devices directly on the pcie bus
3546             # instead of in the root port
3547             if ($winversion == 7) {
3548                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3549             } else {
3550                 $pciaddr = print_pcie_addr("hostpci$i");
3551             }
3552         }else{
3553             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3554         }
3555
3556         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3557         my $romfile = $d->{romfile};
3558
3559         my $xvga = '';
3560         if ($d->{'x-vga'}) {
3561             $xvga = ',x-vga=on';
3562             $kvm_off = 1;
3563             $vga->{type} = 'none' if !defined($conf->{vga});
3564             $gpu_passthrough = 1;
3565
3566             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3567                 $xvga = "";
3568             }
3569         }
3570         my $pcidevices = $d->{pciid};
3571         my $multifunction = 1 if @$pcidevices > 1;
3572         my $sysfspath;
3573         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3574             my $id = $pcidevices->[0]->{id};
3575             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3576             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3577         } elsif ($d->{mdev}) {
3578             warn "ignoring mediated device with multifunction device\n";
3579         }
3580
3581         my $j=0;
3582         foreach my $pcidevice (@$pcidevices) {
3583
3584             my $id = "hostpci$i";
3585             $id .= ".$j" if $multifunction;
3586             my $addr = $pciaddr;
3587             $addr .= ".$j" if $multifunction;
3588             my $devicestr = "vfio-pci";
3589             if ($sysfspath) {
3590                 $devicestr .= ",sysfsdev=$sysfspath";
3591             } else {
3592                 $devicestr .= ",host=$pcidevice->{id}";
3593             }
3594             $devicestr .= ",id=$id$addr";
3595
3596             if($j == 0){
3597                 $devicestr .= "$rombar$xvga";
3598                 $devicestr .= ",multifunction=on" if $multifunction;
3599                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3600             }
3601
3602             push @$devices, '-device', $devicestr;
3603             $j++;
3604         }
3605     }
3606
3607     # usb devices
3608     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3609     push @$devices, @usbdevices if @usbdevices;
3610     # serial devices
3611     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3612         if (my $path = $conf->{"serial$i"}) {
3613             if ($path eq 'socket') {
3614                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3615                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3616                 # On aarch64, serial0 is the UART device. Qemu only allows
3617                 # connecting UART devices via the '-serial' command line, as
3618                 # the device has a fixed slot on the hardware...
3619                 if ($arch eq 'aarch64' && $i == 0) {
3620                     push @$devices, '-serial', "chardev:serial$i";
3621                 } else {
3622                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3623                 }
3624             } else {
3625                 die "no such serial device\n" if ! -c $path;
3626                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3627                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3628             }
3629         }
3630     }
3631
3632     # parallel devices
3633     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3634         if (my $path = $conf->{"parallel$i"}) {
3635             die "no such parallel device\n" if ! -c $path;
3636             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3637             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3638             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3639         }
3640     }
3641
3642
3643     my $sockets = 1;
3644     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3645     $sockets = $conf->{sockets} if  $conf->{sockets};
3646
3647     my $cores = $conf->{cores} || 1;
3648
3649     my $maxcpus = $sockets * $cores;
3650
3651     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3652
3653     my $allowed_vcpus = $cpuinfo->{cpus};
3654
3655     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3656         if ($allowed_vcpus < $maxcpus);
3657
3658     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3659
3660         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3661         for (my $i = 2; $i <= $vcpus; $i++)  {
3662             my $cpustr = print_cpu_device($conf,$i);
3663             push @$cmd, '-device', $cpustr;
3664         }
3665
3666     } else {
3667
3668         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3669     }
3670     push @$cmd, '-nodefaults';
3671
3672     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3673
3674     my $bootindex_hash = {};
3675     my $i = 1;
3676     foreach my $o (split(//, $bootorder)) {
3677         $bootindex_hash->{$o} = $i*100;
3678         $i++;
3679     }
3680
3681     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3682
3683     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3684
3685     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3686
3687     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3688         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3689         my $socket = vnc_socket($vmid);
3690         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3691     } else {
3692         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3693         push @$cmd, '-nographic';
3694     }
3695
3696     # time drift fix
3697     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3698
3699     my $useLocaltime = $conf->{localtime};
3700
3701     if ($winversion >= 5) { # windows
3702         $useLocaltime = 1 if !defined($conf->{localtime});
3703
3704         # use time drift fix when acpi is enabled
3705         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3706             $tdf = 1 if !defined($conf->{tdf});
3707         }
3708     }
3709
3710     if ($winversion >= 6) {
3711         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3712         push @$cmd, '-no-hpet';
3713     }
3714
3715     push @$rtcFlags, 'driftfix=slew' if $tdf;
3716
3717     if (!$kvm) {
3718         push @$machineFlags, 'accel=tcg';
3719     }
3720
3721     if ($machine_type) {
3722         push @$machineFlags, "type=${machine_type}";
3723     }
3724
3725     if ($conf->{startdate}) {
3726         push @$rtcFlags, "base=$conf->{startdate}";
3727     } elsif ($useLocaltime) {
3728         push @$rtcFlags, 'base=localtime';
3729     }
3730
3731     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3732
3733     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3734
3735     push @$cmd, '-S' if $conf->{freeze};
3736
3737     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3738
3739     # enable sound
3740     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3741     #push @$cmd, '-soundhw', 'es1370';
3742     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3743
3744     if (parse_guest_agent($conf)->{enabled}) {
3745         my $qgasocket = qmp_socket($vmid, 1);
3746         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3747         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3748         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3749         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3750     }
3751
3752     my $spice_port;
3753
3754     if ($qxlnum) {
3755         if ($qxlnum > 1) {
3756             if ($winversion){
3757                 for(my $i = 1; $i < $qxlnum; $i++){
3758                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3759                 }
3760             } else {
3761                 # assume other OS works like Linux
3762                 my ($ram, $vram) = ("134217728", "67108864");
3763                 if ($vga->{memory}) {
3764                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3765                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3766                 }
3767                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3768                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3769             }
3770         }
3771
3772         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3773
3774         my $nodename = PVE::INotify::nodename();
3775         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3776         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3777         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3778         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3779         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3780
3781         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3782
3783         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3784         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3785         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3786     }
3787
3788     # enable balloon by default, unless explicitly disabled
3789     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3790         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3791         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3792     }
3793
3794     if ($conf->{watchdog}) {
3795         my $wdopts = parse_watchdog($conf->{watchdog});
3796         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3797         my $watchdog = $wdopts->{model} || 'i6300esb';
3798         push @$devices, '-device', "$watchdog$pciaddr";
3799         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3800     }
3801
3802     my $vollist = [];
3803     my $scsicontroller = {};
3804     my $ahcicontroller = {};
3805     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3806
3807     # Add iscsi initiator name if available
3808     if (my $initiator = get_initiator_name()) {
3809         push @$devices, '-iscsi', "initiator-name=$initiator";
3810     }
3811
3812     foreach_drive($conf, sub {
3813         my ($ds, $drive) = @_;
3814
3815         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3816             push @$vollist, $drive->{file};
3817         }
3818
3819         # ignore efidisk here, already added in bios/fw handling code above
3820         return if $drive->{interface} eq 'efidisk';
3821
3822         $use_virtio = 1 if $ds =~ m/^virtio/;
3823
3824         if (drive_is_cdrom ($drive)) {
3825             if ($bootindex_hash->{d}) {
3826                 $drive->{bootindex} = $bootindex_hash->{d};
3827                 $bootindex_hash->{d} += 1;
3828             }
3829         } else {
3830             if ($bootindex_hash->{c}) {
3831                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3832                 $bootindex_hash->{c} += 1;
3833             }
3834         }
3835
3836         if($drive->{interface} eq 'virtio'){
3837            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3838         }
3839
3840         if ($drive->{interface} eq 'scsi') {
3841
3842             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3843
3844             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3845             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3846
3847             my $iothread = '';
3848             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3849                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3850                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3851             } elsif ($drive->{iothread}) {
3852                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3853             }
3854
3855             my $queues = '';
3856             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3857                 $queues = ",num_queues=$drive->{queues}";
3858             }
3859
3860             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3861             $scsicontroller->{$controller}=1;
3862         }
3863
3864         if ($drive->{interface} eq 'sata') {
3865            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3866            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3867            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3868            $ahcicontroller->{$controller}=1;
3869         }
3870
3871         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3872         push @$devices, '-drive',$drive_cmd;
3873         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3874     });
3875
3876     for (my $i = 0; $i < $MAX_NETS; $i++) {
3877          next if !$conf->{"net$i"};
3878          my $d = parse_net($conf->{"net$i"});
3879          next if !$d;
3880
3881          $use_virtio = 1 if $d->{model} eq 'virtio';
3882
3883          if ($bootindex_hash->{n}) {
3884             $d->{bootindex} = $bootindex_hash->{n};
3885             $bootindex_hash->{n} += 1;
3886          }
3887
3888          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3889          push @$devices, '-netdev', $netdevfull;
3890
3891          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3892          push @$devices, '-device', $netdevicefull;
3893     }
3894
3895     if (!$q35) {
3896         # add pci bridges
3897         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3898            $bridges->{1} = 1;
3899            $bridges->{2} = 1;
3900         }
3901
3902         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3903
3904         while (my ($k, $v) = each %$bridges) {
3905             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3906             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3907         }
3908     }
3909
3910     push @$cmd, @$devices;
3911     push @$cmd, '-rtc', join(',', @$rtcFlags)
3912         if scalar(@$rtcFlags);
3913     push @$cmd, '-machine', join(',', @$machineFlags)
3914         if scalar(@$machineFlags);
3915     push @$cmd, '-global', join(',', @$globalFlags)
3916         if scalar(@$globalFlags);
3917
3918     # add custom args
3919     if ($conf->{args}) {
3920         my $aa = PVE::Tools::split_args($conf->{args});
3921         push @$cmd, @$aa;
3922     }
3923
3924     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3925 }
3926
3927 sub vnc_socket {
3928     my ($vmid) = @_;
3929     return "${var_run_tmpdir}/$vmid.vnc";
3930 }
3931
3932 sub spice_port {
3933     my ($vmid) = @_;
3934
3935     my $res = vm_mon_cmd($vmid, 'query-spice');
3936
3937     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3938 }
3939
3940 sub qmp_socket {
3941     my ($vmid, $qga, $name) = @_;
3942     my $sockettype = $qga ? 'qga' : 'qmp';
3943     my $ext = $name ? '-'.$name : '';
3944     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3945 }
3946
3947 sub pidfile_name {
3948     my ($vmid) = @_;
3949     return "${var_run_tmpdir}/$vmid.pid";
3950 }
3951
3952 sub vm_devices_list {
3953     my ($vmid) = @_;
3954
3955     my $res = vm_mon_cmd($vmid, 'query-pci');
3956     my $devices_to_check = [];
3957     my $devices = {};
3958     foreach my $pcibus (@$res) {
3959         push @$devices_to_check, @{$pcibus->{devices}},
3960     }
3961
3962     while (@$devices_to_check) {
3963         my $to_check = [];
3964         for my $d (@$devices_to_check) {
3965             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3966             next if !$d->{'pci_bridge'};
3967
3968             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3969             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3970         }
3971         $devices_to_check = $to_check;
3972     }
3973
3974     my $resblock = vm_mon_cmd($vmid, 'query-block');
3975     foreach my $block (@$resblock) {
3976         if($block->{device} =~ m/^drive-(\S+)/){
3977                 $devices->{$1} = 1;
3978         }
3979     }
3980
3981     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3982     foreach my $mice (@$resmice) {
3983         if ($mice->{name} eq 'QEMU HID Tablet') {
3984             $devices->{tablet} = 1;
3985             last;
3986         }
3987     }
3988
3989     # for usb devices there is no query-usb
3990     # but we can iterate over the entries in
3991     # qom-list path=/machine/peripheral
3992     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3993     foreach my $per (@$resperipheral) {
3994         if ($per->{name} =~ m/^usb\d+$/) {
3995             $devices->{$per->{name}} = 1;
3996         }
3997     }
3998
3999     return $devices;
4000 }
4001
4002 sub vm_deviceplug {
4003     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4004
4005     my $q35 = machine_type_is_q35($conf);
4006
4007     my $devices_list = vm_devices_list($vmid);
4008     return 1 if defined($devices_list->{$deviceid});
4009
4010     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4011
4012     if ($deviceid eq 'tablet') {
4013
4014         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4015
4016     } elsif ($deviceid eq 'keyboard') {
4017
4018         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4019
4020     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4021
4022         die "usb hotplug currently not reliable\n";
4023         # since we can't reliably hot unplug all added usb devices
4024         # and usb passthrough disables live migration
4025         # we disable usb hotplugging for now
4026         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4027
4028     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4029
4030         qemu_iothread_add($vmid, $deviceid, $device);
4031
4032         qemu_driveadd($storecfg, $vmid, $device);
4033         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4034
4035         qemu_deviceadd($vmid, $devicefull);
4036         eval { qemu_deviceaddverify($vmid, $deviceid); };
4037         if (my $err = $@) {
4038             eval { qemu_drivedel($vmid, $deviceid); };
4039             warn $@ if $@;
4040             die $err;
4041         }
4042
4043     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4044
4045
4046         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4047         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4048         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4049
4050         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4051
4052         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4053             qemu_iothread_add($vmid, $deviceid, $device);
4054             $devicefull .= ",iothread=iothread-$deviceid";
4055         }
4056
4057         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4058             $devicefull .= ",num_queues=$device->{queues}";
4059         }
4060
4061         qemu_deviceadd($vmid, $devicefull);
4062         qemu_deviceaddverify($vmid, $deviceid);
4063
4064     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4065
4066         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4067         qemu_driveadd($storecfg, $vmid, $device);
4068
4069         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4070         eval { qemu_deviceadd($vmid, $devicefull); };
4071         if (my $err = $@) {
4072             eval { qemu_drivedel($vmid, $deviceid); };
4073             warn $@ if $@;
4074             die $err;
4075         }
4076
4077     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4078
4079         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4080
4081         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4082         my $use_old_bios_files = undef;
4083         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4084
4085         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4086         qemu_deviceadd($vmid, $netdevicefull);
4087         eval { qemu_deviceaddverify($vmid, $deviceid); };
4088         if (my $err = $@) {
4089             eval { qemu_netdevdel($vmid, $deviceid); };
4090             warn $@ if $@;
4091             die $err;
4092         }
4093
4094     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4095
4096         my $bridgeid = $2;
4097         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4098         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4099
4100         qemu_deviceadd($vmid, $devicefull);
4101         qemu_deviceaddverify($vmid, $deviceid);
4102
4103     } else {
4104         die "can't hotplug device '$deviceid'\n";
4105     }
4106
4107     return 1;
4108 }
4109
4110 # fixme: this should raise exceptions on error!
4111 sub vm_deviceunplug {
4112     my ($vmid, $conf, $deviceid) = @_;
4113
4114     my $devices_list = vm_devices_list($vmid);
4115     return 1 if !defined($devices_list->{$deviceid});
4116
4117     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4118
4119     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4120
4121         qemu_devicedel($vmid, $deviceid);
4122
4123     } elsif ($deviceid =~ m/^usb\d+$/) {
4124
4125         die "usb hotplug currently not reliable\n";
4126         # when unplugging usb devices this way,
4127         # there may be remaining usb controllers/hubs
4128         # so we disable it for now
4129         qemu_devicedel($vmid, $deviceid);
4130         qemu_devicedelverify($vmid, $deviceid);
4131
4132     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4133
4134         qemu_devicedel($vmid, $deviceid);
4135         qemu_devicedelverify($vmid, $deviceid);
4136         qemu_drivedel($vmid, $deviceid);
4137         qemu_iothread_del($conf, $vmid, $deviceid);
4138
4139     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4140
4141         qemu_devicedel($vmid, $deviceid);
4142         qemu_devicedelverify($vmid, $deviceid);
4143         qemu_iothread_del($conf, $vmid, $deviceid);
4144
4145     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4146
4147         qemu_devicedel($vmid, $deviceid);
4148         qemu_drivedel($vmid, $deviceid);
4149         qemu_deletescsihw($conf, $vmid, $deviceid);
4150
4151     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4152
4153         qemu_devicedel($vmid, $deviceid);
4154         qemu_devicedelverify($vmid, $deviceid);
4155         qemu_netdevdel($vmid, $deviceid);
4156
4157     } else {
4158         die "can't unplug device '$deviceid'\n";
4159     }
4160
4161     return 1;
4162 }
4163
4164 sub qemu_deviceadd {
4165     my ($vmid, $devicefull) = @_;
4166
4167     $devicefull = "driver=".$devicefull;
4168     my %options =  split(/[=,]/, $devicefull);
4169
4170     vm_mon_cmd($vmid, "device_add" , %options);
4171 }
4172
4173 sub qemu_devicedel {
4174     my ($vmid, $deviceid) = @_;
4175
4176     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4177 }
4178
4179 sub qemu_iothread_add {
4180     my($vmid, $deviceid, $device) = @_;
4181
4182     if ($device->{iothread}) {
4183         my $iothreads = vm_iothreads_list($vmid);
4184         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4185     }
4186 }
4187
4188 sub qemu_iothread_del {
4189     my($conf, $vmid, $deviceid) = @_;
4190
4191     my $device = parse_drive($deviceid, $conf->{$deviceid});
4192     if ($device->{iothread}) {
4193         my $iothreads = vm_iothreads_list($vmid);
4194         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4195     }
4196 }
4197
4198 sub qemu_objectadd {
4199     my($vmid, $objectid, $qomtype) = @_;
4200
4201     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4202
4203     return 1;
4204 }
4205
4206 sub qemu_objectdel {
4207     my($vmid, $objectid) = @_;
4208
4209     vm_mon_cmd($vmid, "object-del", id => $objectid);
4210
4211     return 1;
4212 }
4213
4214 sub qemu_driveadd {
4215     my ($storecfg, $vmid, $device) = @_;
4216
4217     my $drive = print_drive_full($storecfg, $vmid, $device);
4218     $drive =~ s/\\/\\\\/g;
4219     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4220
4221     # If the command succeeds qemu prints: "OK"
4222     return 1 if $ret =~ m/OK/s;
4223
4224     die "adding drive failed: $ret\n";
4225 }
4226
4227 sub qemu_drivedel {
4228     my($vmid, $deviceid) = @_;
4229
4230     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4231     $ret =~ s/^\s+//;
4232
4233     return 1 if $ret eq "";
4234
4235     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4236     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4237
4238     die "deleting drive $deviceid failed : $ret\n";
4239 }
4240
4241 sub qemu_deviceaddverify {
4242     my ($vmid, $deviceid) = @_;
4243
4244     for (my $i = 0; $i <= 5; $i++) {
4245          my $devices_list = vm_devices_list($vmid);
4246          return 1 if defined($devices_list->{$deviceid});
4247          sleep 1;
4248     }
4249
4250     die "error on hotplug device '$deviceid'\n";
4251 }
4252
4253
4254 sub qemu_devicedelverify {
4255     my ($vmid, $deviceid) = @_;
4256
4257     # need to verify that the device is correctly removed as device_del
4258     # is async and empty return is not reliable
4259
4260     for (my $i = 0; $i <= 5; $i++) {
4261          my $devices_list = vm_devices_list($vmid);
4262          return 1 if !defined($devices_list->{$deviceid});
4263          sleep 1;
4264     }
4265
4266     die "error on hot-unplugging device '$deviceid'\n";
4267 }
4268
4269 sub qemu_findorcreatescsihw {
4270     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4271
4272     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4273
4274     my $scsihwid="$controller_prefix$controller";
4275     my $devices_list = vm_devices_list($vmid);
4276
4277     if(!defined($devices_list->{$scsihwid})) {
4278         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4279     }
4280
4281     return 1;
4282 }
4283
4284 sub qemu_deletescsihw {
4285     my ($conf, $vmid, $opt) = @_;
4286
4287     my $device = parse_drive($opt, $conf->{$opt});
4288
4289     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4290         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4291         return 1;
4292     }
4293
4294     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4295
4296     my $devices_list = vm_devices_list($vmid);
4297     foreach my $opt (keys %{$devices_list}) {
4298         if (PVE::QemuServer::is_valid_drivename($opt)) {
4299             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4300             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4301                 return 1;
4302             }
4303         }
4304     }
4305
4306     my $scsihwid="scsihw$controller";
4307
4308     vm_deviceunplug($vmid, $conf, $scsihwid);
4309
4310     return 1;
4311 }
4312
4313 sub qemu_add_pci_bridge {
4314     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4315
4316     my $bridges = {};
4317
4318     my $bridgeid;
4319
4320     print_pci_addr($device, $bridges, $arch, $machine_type);
4321
4322     while (my ($k, $v) = each %$bridges) {
4323         $bridgeid = $k;
4324     }
4325     return 1 if !defined($bridgeid) || $bridgeid < 1;
4326
4327     my $bridge = "pci.$bridgeid";
4328     my $devices_list = vm_devices_list($vmid);
4329
4330     if (!defined($devices_list->{$bridge})) {
4331         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4332     }
4333
4334     return 1;
4335 }
4336
4337 sub qemu_set_link_status {
4338     my ($vmid, $device, $up) = @_;
4339
4340     vm_mon_cmd($vmid, "set_link", name => $device,
4341                up => $up ? JSON::true : JSON::false);
4342 }
4343
4344 sub qemu_netdevadd {
4345     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4346
4347     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4348     my %options =  split(/[=,]/, $netdev);
4349
4350     vm_mon_cmd($vmid, "netdev_add",  %options);
4351     return 1;
4352 }
4353
4354 sub qemu_netdevdel {
4355     my ($vmid, $deviceid) = @_;
4356
4357     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4358 }
4359
4360 sub qemu_usb_hotplug {
4361     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4362
4363     return if !$device;
4364
4365     # remove the old one first
4366     vm_deviceunplug($vmid, $conf, $deviceid);
4367
4368     # check if xhci controller is necessary and available
4369     if ($device->{usb3}) {
4370
4371         my $devicelist = vm_devices_list($vmid);
4372
4373         if (!$devicelist->{xhci}) {
4374             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4375             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4376         }
4377     }
4378     my $d = parse_usb_device($device->{host});
4379     $d->{usb3} = $device->{usb3};
4380
4381     # add the new one
4382     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4383 }
4384
4385 sub qemu_cpu_hotplug {
4386     my ($vmid, $conf, $vcpus) = @_;
4387
4388     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4389
4390     my $sockets = 1;
4391     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4392     $sockets = $conf->{sockets} if  $conf->{sockets};
4393     my $cores = $conf->{cores} || 1;
4394     my $maxcpus = $sockets * $cores;
4395
4396     $vcpus = $maxcpus if !$vcpus;
4397
4398     die "you can't add more vcpus than maxcpus\n"
4399         if $vcpus > $maxcpus;
4400
4401     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4402
4403     if ($vcpus < $currentvcpus) {
4404
4405         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4406
4407             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4408                 qemu_devicedel($vmid, "cpu$i");
4409                 my $retry = 0;
4410                 my $currentrunningvcpus = undef;
4411                 while (1) {
4412                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4413                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4414                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4415                     $retry++;
4416                     sleep 1;
4417                 }
4418                 #update conf after each succesfull cpu unplug
4419                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4420                 PVE::QemuConfig->write_config($vmid, $conf);
4421             }
4422         } else {
4423             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4424         }
4425
4426         return;
4427     }
4428
4429     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4430     die "vcpus in running vm does not match its configuration\n"
4431         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4432
4433     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4434
4435         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4436             my $cpustr = print_cpu_device($conf, $i);
4437             qemu_deviceadd($vmid, $cpustr);
4438
4439             my $retry = 0;
4440             my $currentrunningvcpus = undef;
4441             while (1) {
4442                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4443                 last if scalar(@{$currentrunningvcpus}) == $i;
4444                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4445                 sleep 1;
4446                 $retry++;
4447             }
4448             #update conf after each succesfull cpu hotplug
4449             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4450             PVE::QemuConfig->write_config($vmid, $conf);
4451         }
4452     } else {
4453
4454         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4455             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4456         }
4457     }
4458 }
4459
4460 sub qemu_block_set_io_throttle {
4461     my ($vmid, $deviceid,
4462         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4463         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4464         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4465         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4466
4467     return if !check_running($vmid) ;
4468
4469     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4470         bps => int($bps),
4471         bps_rd => int($bps_rd),
4472         bps_wr => int($bps_wr),
4473         iops => int($iops),
4474         iops_rd => int($iops_rd),
4475         iops_wr => int($iops_wr),
4476         bps_max => int($bps_max),
4477         bps_rd_max => int($bps_rd_max),
4478         bps_wr_max => int($bps_wr_max),
4479         iops_max => int($iops_max),
4480         iops_rd_max => int($iops_rd_max),
4481         iops_wr_max => int($iops_wr_max),
4482         bps_max_length => int($bps_max_length),
4483         bps_rd_max_length => int($bps_rd_max_length),
4484         bps_wr_max_length => int($bps_wr_max_length),
4485         iops_max_length => int($iops_max_length),
4486         iops_rd_max_length => int($iops_rd_max_length),
4487         iops_wr_max_length => int($iops_wr_max_length),
4488     );
4489
4490 }
4491
4492 # old code, only used to shutdown old VM after update
4493 sub __read_avail {
4494     my ($fh, $timeout) = @_;
4495
4496     my $sel = new IO::Select;
4497     $sel->add($fh);
4498
4499     my $res = '';
4500     my $buf;
4501
4502     my @ready;
4503     while (scalar (@ready = $sel->can_read($timeout))) {
4504         my $count;
4505         if ($count = $fh->sysread($buf, 8192)) {
4506             if ($buf =~ /^(.*)\(qemu\) $/s) {
4507                 $res .= $1;
4508                 last;
4509             } else {
4510                 $res .= $buf;
4511             }
4512         } else {
4513             if (!defined($count)) {
4514                 die "$!\n";
4515             }
4516             last;
4517         }
4518     }
4519
4520     die "monitor read timeout\n" if !scalar(@ready);
4521
4522     return $res;
4523 }
4524
4525 sub qemu_block_resize {
4526     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4527
4528     my $running = check_running($vmid);
4529
4530     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4531
4532     return if !$running;
4533
4534     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4535
4536 }
4537
4538 sub qemu_volume_snapshot {
4539     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4540
4541     my $running = check_running($vmid);
4542
4543     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4544         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4545     } else {
4546         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4547     }
4548 }
4549
4550 sub qemu_volume_snapshot_delete {
4551     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4552
4553     my $running = check_running($vmid);
4554
4555     if($running) {
4556
4557         $running = undef;
4558         my $conf = PVE::QemuConfig->load_config($vmid);
4559         foreach_drive($conf, sub {
4560             my ($ds, $drive) = @_;
4561             $running = 1 if $drive->{file} eq $volid;
4562         });
4563     }
4564
4565     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4566         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4567     } else {
4568         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4569     }
4570 }
4571
4572 sub set_migration_caps {
4573     my ($vmid) = @_;
4574
4575     my $cap_ref = [];
4576
4577     my $enabled_cap = {
4578         "auto-converge" => 1,
4579         "xbzrle" => 1,
4580         "x-rdma-pin-all" => 0,
4581         "zero-blocks" => 0,
4582         "compress" => 0
4583     };
4584
4585     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4586
4587     for my $supported_capability (@$supported_capabilities) {
4588         push @$cap_ref, {
4589             capability => $supported_capability->{capability},
4590             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4591         };
4592     }
4593
4594     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4595 }
4596
4597 my $fast_plug_option = {
4598     'lock' => 1,
4599     'name' => 1,
4600     'onboot' => 1,
4601     'shares' => 1,
4602     'startup' => 1,
4603     'description' => 1,
4604     'protection' => 1,
4605     'vmstatestorage' => 1,
4606     'hookscript' => 1,
4607 };
4608
4609 # hotplug changes in [PENDING]
4610 # $selection hash can be used to only apply specified options, for
4611 # example: { cores => 1 } (only apply changed 'cores')
4612 # $errors ref is used to return error messages
4613 sub vmconfig_hotplug_pending {
4614     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4615
4616     my $defaults = load_defaults();
4617     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4618
4619     # commit values which do not have any impact on running VM first
4620     # Note: those option cannot raise errors, we we do not care about
4621     # $selection and always apply them.
4622
4623     my $add_error = sub {
4624         my ($opt, $msg) = @_;
4625         $errors->{$opt} = "hotplug problem - $msg";
4626     };
4627
4628     my $changes = 0;
4629     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4630         if ($fast_plug_option->{$opt}) {
4631             $conf->{$opt} = $conf->{pending}->{$opt};
4632             delete $conf->{pending}->{$opt};
4633             $changes = 1;
4634         }
4635     }
4636
4637     if ($changes) {
4638         PVE::QemuConfig->write_config($vmid, $conf);
4639         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4640     }
4641
4642     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4643
4644     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4645     while (my ($opt, $force) = each %$pending_delete_hash) {
4646         next if $selection && !$selection->{$opt};
4647         eval {
4648             if ($opt eq 'hotplug') {
4649                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4650             } elsif ($opt eq 'tablet') {
4651                 die "skip\n" if !$hotplug_features->{usb};
4652                 if ($defaults->{tablet}) {
4653                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4654                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4655                         if $arch eq 'aarch64';
4656                 } else {
4657                     vm_deviceunplug($vmid, $conf, 'tablet');
4658                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4659                 }
4660             } elsif ($opt =~ m/^usb\d+/) {
4661                 die "skip\n";
4662                 # since we cannot reliably hot unplug usb devices
4663                 # we are disabling it
4664                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4665                 vm_deviceunplug($vmid, $conf, $opt);
4666             } elsif ($opt eq 'vcpus') {
4667                 die "skip\n" if !$hotplug_features->{cpu};
4668                 qemu_cpu_hotplug($vmid, $conf, undef);
4669             } elsif ($opt eq 'balloon') {
4670                 # enable balloon device is not hotpluggable
4671                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4672                 # here we reset the ballooning value to memory
4673                 my $balloon = $conf->{memory} || $defaults->{memory};
4674                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4675             } elsif ($fast_plug_option->{$opt}) {
4676                 # do nothing
4677             } elsif ($opt =~ m/^net(\d+)$/) {
4678                 die "skip\n" if !$hotplug_features->{network};
4679                 vm_deviceunplug($vmid, $conf, $opt);
4680             } elsif (is_valid_drivename($opt)) {
4681                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4682                 vm_deviceunplug($vmid, $conf, $opt);
4683                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4684             } elsif ($opt =~ m/^memory$/) {
4685                 die "skip\n" if !$hotplug_features->{memory};
4686                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4687             } elsif ($opt eq 'cpuunits') {
4688                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4689             } elsif ($opt eq 'cpulimit') {
4690                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4691             } else {
4692                 die "skip\n";
4693             }
4694         };
4695         if (my $err = $@) {
4696             &$add_error($opt, $err) if $err ne "skip\n";
4697         } else {
4698             # save new config if hotplug was successful
4699             delete $conf->{$opt};
4700             vmconfig_undelete_pending_option($conf, $opt);
4701             PVE::QemuConfig->write_config($vmid, $conf);
4702             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4703         }
4704     }
4705
4706     my $apply_pending_cloudinit;
4707     $apply_pending_cloudinit = sub {
4708         my ($key, $value) = @_;
4709         $apply_pending_cloudinit = sub {}; # once is enough
4710
4711         my @cloudinit_opts = keys %$confdesc_cloudinit;
4712         foreach my $opt (keys %{$conf->{pending}}) {
4713             next if !grep { $_ eq $opt } @cloudinit_opts;
4714             $conf->{$opt} = delete $conf->{pending}->{$opt};
4715         }
4716
4717         my $new_conf = { %$conf };
4718         $new_conf->{$key} = $value;
4719         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4720     };
4721
4722     foreach my $opt (keys %{$conf->{pending}}) {
4723         next if $selection && !$selection->{$opt};
4724         my $value = $conf->{pending}->{$opt};
4725         eval {
4726             if ($opt eq 'hotplug') {
4727                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4728             } elsif ($opt eq 'tablet') {
4729                 die "skip\n" if !$hotplug_features->{usb};
4730                 if ($value == 1) {
4731                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4732                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4733                         if $arch eq 'aarch64';
4734                 } elsif ($value == 0) {
4735                     vm_deviceunplug($vmid, $conf, 'tablet');
4736                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4737                 }
4738             } elsif ($opt =~ m/^usb\d+$/) {
4739                 die "skip\n";
4740                 # since we cannot reliably hot unplug usb devices
4741                 # we are disabling it
4742                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4743                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4744                 die "skip\n" if !$d;
4745                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4746             } elsif ($opt eq 'vcpus') {
4747                 die "skip\n" if !$hotplug_features->{cpu};
4748                 qemu_cpu_hotplug($vmid, $conf, $value);
4749             } elsif ($opt eq 'balloon') {
4750                 # enable/disable balloning device is not hotpluggable
4751                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4752                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4753                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4754
4755                 # allow manual ballooning if shares is set to zero
4756                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4757                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4758                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4759                 }
4760             } elsif ($opt =~ m/^net(\d+)$/) {
4761                 # some changes can be done without hotplug
4762                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4763                                     $vmid, $opt, $value, $arch, $machine_type);
4764             } elsif (is_valid_drivename($opt)) {
4765                 # some changes can be done without hotplug
4766                 my $drive = parse_drive($opt, $value);
4767                 if (drive_is_cloudinit($drive)) {
4768                     &$apply_pending_cloudinit($opt, $value);
4769                 }
4770                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4771                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4772             } elsif ($opt =~ m/^memory$/) { #dimms
4773                 die "skip\n" if !$hotplug_features->{memory};
4774                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4775             } elsif ($opt eq 'cpuunits') {
4776                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4777             } elsif ($opt eq 'cpulimit') {
4778                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4779                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4780             } else {
4781                 die "skip\n";  # skip non-hot-pluggable options
4782             }
4783         };
4784         if (my $err = $@) {
4785             &$add_error($opt, $err) if $err ne "skip\n";
4786         } else {
4787             # save new config if hotplug was successful
4788             $conf->{$opt} = $value;
4789             delete $conf->{pending}->{$opt};
4790             PVE::QemuConfig->write_config($vmid, $conf);
4791             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4792         }
4793     }
4794 }
4795
4796 sub try_deallocate_drive {
4797     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4798
4799     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4800         my $volid = $drive->{file};
4801         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4802             my $sid = PVE::Storage::parse_volume_id($volid);
4803             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4804
4805             # check if the disk is really unused
4806             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4807                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4808             PVE::Storage::vdisk_free($storecfg, $volid);
4809             return 1;
4810         } else {
4811             # If vm is not owner of this disk remove from config
4812             return 1;
4813         }
4814     }
4815
4816     return undef;
4817 }
4818
4819 sub vmconfig_delete_or_detach_drive {
4820     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4821
4822     my $drive = parse_drive($opt, $conf->{$opt});
4823
4824     my $rpcenv = PVE::RPCEnvironment::get();
4825     my $authuser = $rpcenv->get_user();
4826
4827     if ($force) {
4828         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4829         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4830     } else {
4831         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4832     }
4833 }
4834
4835 sub vmconfig_apply_pending {
4836     my ($vmid, $conf, $storecfg) = @_;
4837
4838     # cold plug
4839
4840     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4841     while (my ($opt, $force) = each %$pending_delete_hash) {
4842         die "internal error" if $opt =~ m/^unused/;
4843         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4844         if (!defined($conf->{$opt})) {
4845             vmconfig_undelete_pending_option($conf, $opt);
4846             PVE::QemuConfig->write_config($vmid, $conf);
4847         } elsif (is_valid_drivename($opt)) {
4848             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4849             vmconfig_undelete_pending_option($conf, $opt);
4850             delete $conf->{$opt};
4851             PVE::QemuConfig->write_config($vmid, $conf);
4852         } else {
4853             vmconfig_undelete_pending_option($conf, $opt);
4854             delete $conf->{$opt};
4855             PVE::QemuConfig->write_config($vmid, $conf);
4856         }
4857     }
4858
4859     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4860
4861     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4862         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4863
4864         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4865             # skip if nothing changed
4866         } elsif (is_valid_drivename($opt)) {
4867             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4868                 if defined($conf->{$opt});
4869             $conf->{$opt} = $conf->{pending}->{$opt};
4870         } else {
4871             $conf->{$opt} = $conf->{pending}->{$opt};
4872         }
4873
4874         delete $conf->{pending}->{$opt};
4875         PVE::QemuConfig->write_config($vmid, $conf);
4876     }
4877 }
4878
4879 my $safe_num_ne = sub {
4880     my ($a, $b) = @_;
4881
4882     return 0 if !defined($a) && !defined($b);
4883     return 1 if !defined($a);
4884     return 1 if !defined($b);
4885
4886     return $a != $b;
4887 };
4888
4889 my $safe_string_ne = sub {
4890     my ($a, $b) = @_;
4891
4892     return 0 if !defined($a) && !defined($b);
4893     return 1 if !defined($a);
4894     return 1 if !defined($b);
4895
4896     return $a ne $b;
4897 };
4898
4899 sub vmconfig_update_net {
4900     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4901
4902     my $newnet = parse_net($value);
4903
4904     if ($conf->{$opt}) {
4905         my $oldnet = parse_net($conf->{$opt});
4906
4907         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4908             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4909             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4910             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4911
4912             # for non online change, we try to hot-unplug
4913             die "skip\n" if !$hotplug;
4914             vm_deviceunplug($vmid, $conf, $opt);
4915         } else {
4916
4917             die "internal error" if $opt !~ m/net(\d+)/;
4918             my $iface = "tap${vmid}i$1";
4919
4920             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4921                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4922                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4923                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4924                 PVE::Network::tap_unplug($iface);
4925                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4926             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4927                 # Rate can be applied on its own but any change above needs to
4928                 # include the rate in tap_plug since OVS resets everything.
4929                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4930             }
4931
4932             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4933                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4934             }
4935
4936             return 1;
4937         }
4938     }
4939
4940     if ($hotplug) {
4941         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4942     } else {
4943         die "skip\n";
4944     }
4945 }
4946
4947 sub vmconfig_update_disk {
4948     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4949
4950     # fixme: do we need force?
4951
4952     my $drive = parse_drive($opt, $value);
4953
4954     if ($conf->{$opt}) {
4955
4956         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4957
4958             my $media = $drive->{media} || 'disk';
4959             my $oldmedia = $old_drive->{media} || 'disk';
4960             die "unable to change media type\n" if $media ne $oldmedia;
4961
4962             if (!drive_is_cdrom($old_drive)) {
4963
4964                 if ($drive->{file} ne $old_drive->{file}) {
4965
4966                     die "skip\n" if !$hotplug;
4967
4968                     # unplug and register as unused
4969                     vm_deviceunplug($vmid, $conf, $opt);
4970                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4971
4972                 } else {
4973                     # update existing disk
4974
4975                     # skip non hotpluggable value
4976                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4977                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4978                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4979                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4980                         die "skip\n";
4981                     }
4982
4983                     # apply throttle
4984                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4985                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4986                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4987                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4988                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4989                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4990                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4991                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4992                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4993                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4994                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4995                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4996                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4997                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4998                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4999                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5000                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5001                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5002
5003                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5004                                                    ($drive->{mbps} || 0)*1024*1024,
5005                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5006                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5007                                                    $drive->{iops} || 0,
5008                                                    $drive->{iops_rd} || 0,
5009                                                    $drive->{iops_wr} || 0,
5010                                                    ($drive->{mbps_max} || 0)*1024*1024,
5011                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5012                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5013                                                    $drive->{iops_max} || 0,
5014                                                    $drive->{iops_rd_max} || 0,
5015                                                    $drive->{iops_wr_max} || 0,
5016                                                    $drive->{bps_max_length} || 1,
5017                                                    $drive->{bps_rd_max_length} || 1,
5018                                                    $drive->{bps_wr_max_length} || 1,
5019                                                    $drive->{iops_max_length} || 1,
5020                                                    $drive->{iops_rd_max_length} || 1,
5021                                                    $drive->{iops_wr_max_length} || 1);
5022
5023                     }
5024
5025                     return 1;
5026                 }
5027
5028             } else { # cdrom
5029
5030                 if ($drive->{file} eq 'none') {
5031                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5032                     if (drive_is_cloudinit($old_drive)) {
5033                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5034                     }
5035                 } else {
5036                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5037                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5038                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5039                 }
5040
5041                 return 1;
5042             }
5043         }
5044     }
5045
5046     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5047     # hotplug new disks
5048     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5049     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5050 }
5051
5052 sub vm_start {
5053     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5054         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5055
5056     PVE::QemuConfig->lock_config($vmid, sub {
5057         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5058
5059         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5060
5061         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5062
5063         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5064
5065         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5066             vmconfig_apply_pending($vmid, $conf, $storecfg);
5067             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5068         }
5069
5070         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5071
5072         my $defaults = load_defaults();
5073
5074         # set environment variable useful inside network script
5075         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5076
5077         my $local_volumes = {};
5078
5079         if ($targetstorage) {
5080             foreach_drive($conf, sub {
5081                 my ($ds, $drive) = @_;
5082
5083                 return if drive_is_cdrom($drive);
5084
5085                 my $volid = $drive->{file};
5086
5087                 return if !$volid;
5088
5089                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5090
5091                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5092                 return if $scfg->{shared};
5093                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5094             });
5095
5096             my $format = undef;
5097
5098             foreach my $opt (sort keys %$local_volumes) {
5099
5100                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5101                 my $drive = parse_drive($opt, $conf->{$opt});
5102
5103                 #if remote storage is specified, use default format
5104                 if ($targetstorage && $targetstorage ne "1") {
5105                     $storeid = $targetstorage;
5106                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5107                     $format = $defFormat;
5108                 } else {
5109                     #else we use same format than original
5110                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5111                     $format = qemu_img_format($scfg, $volid);
5112                 }
5113
5114                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5115                 my $newdrive = $drive;
5116                 $newdrive->{format} = $format;
5117                 $newdrive->{file} = $newvolid;
5118                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5119                 $local_volumes->{$opt} = $drivestr;
5120                 #pass drive to conf for command line
5121                 $conf->{$opt} = $drivestr;
5122             }
5123         }
5124
5125         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5126
5127         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5128
5129         my $migrate_port = 0;
5130         my $migrate_uri;
5131         if ($statefile) {
5132             if ($statefile eq 'tcp') {
5133                 my $localip = "localhost";
5134                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5135                 my $nodename = PVE::INotify::nodename();
5136
5137                 if (!defined($migration_type)) {
5138                     if (defined($datacenterconf->{migration}->{type})) {
5139                         $migration_type = $datacenterconf->{migration}->{type};
5140                     } else {
5141                         $migration_type = 'secure';
5142                     }
5143                 }
5144
5145                 if ($migration_type eq 'insecure') {
5146                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5147                     if ($migrate_network_addr) {
5148                         $localip = $migrate_network_addr;
5149                     } else {
5150                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5151                     }
5152
5153                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5154                 }
5155
5156                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5157                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5158                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5159                 push @$cmd, '-incoming', $migrate_uri;
5160                 push @$cmd, '-S';
5161
5162             } elsif ($statefile eq 'unix') {
5163                 # should be default for secure migrations as a ssh TCP forward
5164                 # tunnel is not deterministic reliable ready and fails regurarly
5165                 # to set up in time, so use UNIX socket forwards
5166                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5167                 unlink $socket_addr;
5168
5169                 $migrate_uri = "unix:$socket_addr";
5170
5171                 push @$cmd, '-incoming', $migrate_uri;
5172                 push @$cmd, '-S';
5173
5174             } else {
5175                 push @$cmd, '-loadstate', $statefile;
5176             }
5177         } elsif ($paused) {
5178             push @$cmd, '-S';
5179         }
5180
5181         # host pci devices
5182         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5183           my $d = parse_hostpci($conf->{"hostpci$i"});
5184           next if !$d;
5185           my $pcidevices = $d->{pciid};
5186           foreach my $pcidevice (@$pcidevices) {
5187                 my $pciid = $pcidevice->{id};
5188
5189                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5190                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5191                 die "no pci device info for device '$pciid'\n" if !$info;
5192
5193                 if ($d->{mdev}) {
5194                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5195                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5196                 } else {
5197                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5198                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5199                     die "can't reset pci device '$pciid'\n"
5200                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5201                 }
5202           }
5203         }
5204
5205         PVE::Storage::activate_volumes($storecfg, $vollist);
5206
5207         if (!check_running($vmid, 1)) {
5208             eval {
5209                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5210                     outfunc => sub {}, errfunc => sub {});
5211             };
5212         }
5213
5214         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5215                                                   : $defaults->{cpuunits};
5216
5217         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5218         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5219
5220         my %properties = (
5221             Slice => 'qemu.slice',
5222             KillMode => 'none',
5223             CPUShares => $cpuunits
5224         );
5225
5226         if (my $cpulimit = $conf->{cpulimit}) {
5227             $properties{CPUQuota} = int($cpulimit * 100);
5228         }
5229         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5230
5231         my $run_qemu = sub {
5232             PVE::Tools::run_fork sub {
5233                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5234                 run_command($cmd, %run_params);
5235             };
5236         };
5237
5238         if ($conf->{hugepages}) {
5239
5240             my $code = sub {
5241                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5242                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5243
5244                 PVE::QemuServer::Memory::hugepages_mount();
5245                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5246
5247                 eval { $run_qemu->() };
5248                 if (my $err = $@) {
5249                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5250                     die $err;
5251                 }
5252
5253                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5254             };
5255             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5256
5257         } else {
5258             eval { $run_qemu->() };
5259         }
5260
5261         if (my $err = $@) {
5262             # deactivate volumes if start fails
5263             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5264             die "start failed: $err";
5265         }
5266
5267         print "migration listens on $migrate_uri\n" if $migrate_uri;
5268
5269         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5270             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5271             warn $@ if $@;
5272         }
5273
5274         #start nbd server for storage migration
5275         if ($targetstorage) {
5276             my $nodename = PVE::INotify::nodename();
5277             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5278             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5279             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5280             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5281
5282             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5283
5284             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5285
5286             foreach my $opt (sort keys %$local_volumes) {
5287                 my $volid = $local_volumes->{$opt};
5288                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5289                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5290                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5291             }
5292         }
5293
5294         if ($migratedfrom) {
5295             eval {
5296                 set_migration_caps($vmid);
5297             };
5298             warn $@ if $@;
5299
5300             if ($spice_port) {
5301                 print "spice listens on port $spice_port\n";
5302                 if ($spice_ticket) {
5303                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5304                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5305                 }
5306             }
5307
5308         } else {
5309             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5310                 if !$statefile && $conf->{balloon};
5311
5312             foreach my $opt (keys %$conf) {
5313                 next if $opt !~  m/^net\d+$/;
5314                 my $nicconf = parse_net($conf->{$opt});
5315                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5316             }
5317         }
5318
5319         vm_mon_cmd_nocheck($vmid, 'qom-set',
5320                     path => "machine/peripheral/balloon0",
5321                     property => "guest-stats-polling-interval",
5322                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5323
5324         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5325     });
5326 }
5327
5328 sub vm_mon_cmd {
5329     my ($vmid, $execute, %params) = @_;
5330
5331     my $cmd = { execute => $execute, arguments => \%params };
5332     vm_qmp_command($vmid, $cmd);
5333 }
5334
5335 sub vm_mon_cmd_nocheck {
5336     my ($vmid, $execute, %params) = @_;
5337
5338     my $cmd = { execute => $execute, arguments => \%params };
5339     vm_qmp_command($vmid, $cmd, 1);
5340 }
5341
5342 sub vm_qmp_command {
5343     my ($vmid, $cmd, $nocheck) = @_;
5344
5345     my $res;
5346
5347     my $timeout;
5348     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5349         $timeout = $cmd->{arguments}->{timeout};
5350         delete $cmd->{arguments}->{timeout};
5351     }
5352
5353     eval {
5354         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5355         my $sname = qmp_socket($vmid);
5356         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5357             my $qmpclient = PVE::QMPClient->new();
5358
5359             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5360         } else {
5361             die "unable to open monitor socket\n";
5362         }
5363     };
5364     if (my $err = $@) {
5365         syslog("err", "VM $vmid qmp command failed - $err");
5366         die $err;
5367     }
5368
5369     return $res;
5370 }
5371
5372 sub vm_human_monitor_command {
5373     my ($vmid, $cmdline) = @_;
5374
5375     my $res;
5376
5377     my $cmd = {
5378         execute => 'human-monitor-command',
5379         arguments => { 'command-line' => $cmdline},
5380     };
5381
5382     return vm_qmp_command($vmid, $cmd);
5383 }
5384
5385 sub vm_commandline {
5386     my ($storecfg, $vmid, $snapname) = @_;
5387
5388     my $conf = PVE::QemuConfig->load_config($vmid);
5389
5390     if ($snapname) {
5391         my $snapshot = $conf->{snapshots}->{$snapname};
5392         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5393
5394         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5395
5396         $conf = $snapshot;
5397     }
5398
5399     my $defaults = load_defaults();
5400
5401     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5402
5403     return PVE::Tools::cmd2string($cmd);
5404 }
5405
5406 sub vm_reset {
5407     my ($vmid, $skiplock) = @_;
5408
5409     PVE::QemuConfig->lock_config($vmid, sub {
5410
5411         my $conf = PVE::QemuConfig->load_config($vmid);
5412
5413         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5414
5415         vm_mon_cmd($vmid, "system_reset");
5416     });
5417 }
5418
5419 sub get_vm_volumes {
5420     my ($conf) = @_;
5421
5422     my $vollist = [];
5423     foreach_volid($conf, sub {
5424         my ($volid, $attr) = @_;
5425
5426         return if $volid =~ m|^/|;
5427
5428         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5429         return if !$sid;
5430
5431         push @$vollist, $volid;
5432     });
5433
5434     return $vollist;
5435 }
5436
5437 sub vm_stop_cleanup {
5438     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5439
5440     eval {
5441
5442         if (!$keepActive) {
5443             my $vollist = get_vm_volumes($conf);
5444             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5445         }
5446
5447         foreach my $ext (qw(mon qmp pid vnc qga)) {
5448             unlink "/var/run/qemu-server/${vmid}.$ext";
5449         }
5450
5451         foreach my $key (keys %$conf) {
5452             next if $key !~ m/^hostpci(\d+)$/;
5453             my $hostpciindex = $1;
5454             my $d = parse_hostpci($conf->{$key});
5455             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5456
5457             foreach my $pci (@{$d->{pciid}}) {
5458                 my $pciid = $pci->{id};
5459                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5460             }
5461         }
5462
5463         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5464     };
5465     warn $@ if $@; # avoid errors - just warn
5466 }
5467
5468 # Note: use $nockeck to skip tests if VM configuration file exists.
5469 # We need that when migration VMs to other nodes (files already moved)
5470 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5471 sub vm_stop {
5472     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5473
5474     $force = 1 if !defined($force) && !$shutdown;
5475
5476     if ($migratedfrom){
5477         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5478         kill 15, $pid if $pid;
5479         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5480         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5481         return;
5482     }
5483
5484     PVE::QemuConfig->lock_config($vmid, sub {
5485
5486         my $pid = check_running($vmid, $nocheck);
5487         return if !$pid;
5488
5489         my $conf;
5490         if (!$nocheck) {
5491             $conf = PVE::QemuConfig->load_config($vmid);
5492             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5493             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5494                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5495                 $timeout = $opts->{down} if $opts->{down};
5496             }
5497             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5498         }
5499
5500         $timeout = 60 if !defined($timeout);
5501
5502         eval {
5503             if ($shutdown) {
5504                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5505                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5506                 } else {
5507                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5508                 }
5509             } else {
5510                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5511             }
5512         };
5513         my $err = $@;
5514
5515         if (!$err) {
5516             my $count = 0;
5517             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5518                 $count++;
5519                 sleep 1;
5520             }
5521
5522             if ($count >= $timeout) {
5523                 if ($force) {
5524                     warn "VM still running - terminating now with SIGTERM\n";
5525                     kill 15, $pid;
5526                 } else {
5527                     die "VM quit/powerdown failed - got timeout\n";
5528                 }
5529             } else {
5530                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5531                 return;
5532             }
5533         } else {
5534             if ($force) {
5535                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5536                 kill 15, $pid;
5537             } else {
5538                 die "VM quit/powerdown failed\n";
5539             }
5540         }
5541
5542         # wait again
5543         $timeout = 10;
5544
5545         my $count = 0;
5546         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5547             $count++;
5548             sleep 1;
5549         }
5550
5551         if ($count >= $timeout) {
5552             warn "VM still running - terminating now with SIGKILL\n";
5553             kill 9, $pid;
5554             sleep 1;
5555         }
5556
5557         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5558    });
5559 }
5560
5561 sub vm_suspend {
5562     my ($vmid, $skiplock) = @_;
5563
5564     PVE::QemuConfig->lock_config($vmid, sub {
5565
5566         my $conf = PVE::QemuConfig->load_config($vmid);
5567
5568         PVE::QemuConfig->check_lock($conf)
5569             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5570
5571         vm_mon_cmd($vmid, "stop");
5572     });
5573 }
5574
5575 sub vm_resume {
5576     my ($vmid, $skiplock, $nocheck) = @_;
5577
5578     PVE::QemuConfig->lock_config($vmid, sub {
5579
5580         my $res = vm_mon_cmd($vmid, 'query-status');
5581         my $resume_cmd = 'cont';
5582
5583         if ($res->{status} && $res->{status} eq 'suspended') {
5584             $resume_cmd = 'system_wakeup';
5585         }
5586
5587         if (!$nocheck) {
5588
5589             my $conf = PVE::QemuConfig->load_config($vmid);
5590
5591             PVE::QemuConfig->check_lock($conf)
5592                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5593
5594             vm_mon_cmd($vmid, $resume_cmd);
5595
5596         } else {
5597             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5598         }
5599     });
5600 }
5601
5602 sub vm_sendkey {
5603     my ($vmid, $skiplock, $key) = @_;
5604
5605     PVE::QemuConfig->lock_config($vmid, sub {
5606
5607         my $conf = PVE::QemuConfig->load_config($vmid);
5608
5609         # there is no qmp command, so we use the human monitor command
5610         vm_human_monitor_command($vmid, "sendkey $key");
5611     });
5612 }
5613
5614 sub vm_destroy {
5615     my ($storecfg, $vmid, $skiplock) = @_;
5616
5617     PVE::QemuConfig->lock_config($vmid, sub {
5618
5619         my $conf = PVE::QemuConfig->load_config($vmid);
5620
5621         if (!check_running($vmid)) {
5622             destroy_vm($storecfg, $vmid, undef, $skiplock);
5623         } else {
5624             die "VM $vmid is running - destroy failed\n";
5625         }
5626     });
5627 }
5628
5629 # vzdump restore implementaion
5630
5631 sub tar_archive_read_firstfile {
5632     my $archive = shift;
5633
5634     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5635
5636     # try to detect archive type first
5637     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5638         die "unable to open file '$archive'\n";
5639     my $firstfile = <$fh>;
5640     kill 15, $pid;
5641     close $fh;
5642
5643     die "ERROR: archive contaions no data\n" if !$firstfile;
5644     chomp $firstfile;
5645
5646     return $firstfile;
5647 }
5648
5649 sub tar_restore_cleanup {
5650     my ($storecfg, $statfile) = @_;
5651
5652     print STDERR "starting cleanup\n";
5653
5654     if (my $fd = IO::File->new($statfile, "r")) {
5655         while (defined(my $line = <$fd>)) {
5656             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5657                 my $volid = $2;
5658                 eval {
5659                     if ($volid =~ m|^/|) {
5660                         unlink $volid || die 'unlink failed\n';
5661                     } else {
5662                         PVE::Storage::vdisk_free($storecfg, $volid);
5663                     }
5664                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5665                 };
5666                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5667             } else {
5668                 print STDERR "unable to parse line in statfile - $line";
5669             }
5670         }
5671         $fd->close();
5672     }
5673 }
5674
5675 sub restore_archive {
5676     my ($archive, $vmid, $user, $opts) = @_;
5677
5678     my $format = $opts->{format};
5679     my $comp;
5680
5681     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5682         $format = 'tar' if !$format;
5683         $comp = 'gzip';
5684     } elsif ($archive =~ m/\.tar$/) {
5685         $format = 'tar' if !$format;
5686     } elsif ($archive =~ m/.tar.lzo$/) {
5687         $format = 'tar' if !$format;
5688         $comp = 'lzop';
5689     } elsif ($archive =~ m/\.vma$/) {
5690         $format = 'vma' if !$format;
5691     } elsif ($archive =~ m/\.vma\.gz$/) {
5692         $format = 'vma' if !$format;
5693         $comp = 'gzip';
5694     } elsif ($archive =~ m/\.vma\.lzo$/) {
5695         $format = 'vma' if !$format;
5696         $comp = 'lzop';
5697     } else {
5698         $format = 'vma' if !$format; # default
5699     }
5700
5701     # try to detect archive format
5702     if ($format eq 'tar') {
5703         return restore_tar_archive($archive, $vmid, $user, $opts);
5704     } else {
5705         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5706     }
5707 }
5708
5709 sub restore_update_config_line {
5710     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5711
5712     return if $line =~ m/^\#qmdump\#/;
5713     return if $line =~ m/^\#vzdump\#/;
5714     return if $line =~ m/^lock:/;
5715     return if $line =~ m/^unused\d+:/;
5716     return if $line =~ m/^parent:/;
5717     return if $line =~ m/^template:/; # restored VM is never a template
5718
5719     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5720     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5721         # try to convert old 1.X settings
5722         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5723         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5724             my ($model, $macaddr) = split(/\=/, $devconfig);
5725             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5726             my $net = {
5727                 model => $model,
5728                 bridge => "vmbr$ind",
5729                 macaddr => $macaddr,
5730             };
5731             my $netstr = print_net($net);
5732
5733             print $outfd "net$cookie->{netcount}: $netstr\n";
5734             $cookie->{netcount}++;
5735         }
5736     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5737         my ($id, $netstr) = ($1, $2);
5738         my $net = parse_net($netstr);
5739         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5740         $netstr = print_net($net);
5741         print $outfd "$id: $netstr\n";
5742     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5743         my $virtdev = $1;
5744         my $value = $3;
5745         my $di = parse_drive($virtdev, $value);
5746         if (defined($di->{backup}) && !$di->{backup}) {
5747             print $outfd "#$line";
5748         } elsif ($map->{$virtdev}) {
5749             delete $di->{format}; # format can change on restore
5750             $di->{file} = $map->{$virtdev};
5751             $value = print_drive($vmid, $di);
5752             print $outfd "$virtdev: $value\n";
5753         } else {
5754             print $outfd $line;
5755         }
5756     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5757         my $vmgenid = $1;
5758         if ($vmgenid ne '0') {
5759             # always generate a new vmgenid if there was a valid one setup
5760             $vmgenid = generate_uuid();
5761         }
5762         print $outfd "vmgenid: $vmgenid\n";
5763     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5764         my ($uuid, $uuid_str);
5765         UUID::generate($uuid);
5766         UUID::unparse($uuid, $uuid_str);
5767         my $smbios1 = parse_smbios1($2);
5768         $smbios1->{uuid} = $uuid_str;
5769         print $outfd $1.print_smbios1($smbios1)."\n";
5770     } else {
5771         print $outfd $line;
5772     }
5773 }
5774
5775 sub scan_volids {
5776     my ($cfg, $vmid) = @_;
5777
5778     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5779
5780     my $volid_hash = {};
5781     foreach my $storeid (keys %$info) {
5782         foreach my $item (@{$info->{$storeid}}) {
5783             next if !($item->{volid} && $item->{size});
5784             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5785             $volid_hash->{$item->{volid}} = $item;
5786         }
5787     }
5788
5789     return $volid_hash;
5790 }
5791
5792 sub is_volume_in_use {
5793     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5794
5795     my $path = PVE::Storage::path($storecfg, $volid);
5796
5797     my $scan_config = sub {
5798         my ($cref, $snapname) = @_;
5799
5800         foreach my $key (keys %$cref) {
5801             my $value = $cref->{$key};
5802             if (is_valid_drivename($key)) {
5803                 next if $skip_drive && $key eq $skip_drive;
5804                 my $drive = parse_drive($key, $value);
5805                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5806                 return 1 if $volid eq $drive->{file};
5807                 if ($drive->{file} =~ m!^/!) {
5808                     return 1 if $drive->{file} eq $path;
5809                 } else {
5810                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5811                     next if !$storeid;
5812                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5813                     next if !$scfg;
5814                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5815                 }
5816             }
5817         }
5818
5819         return 0;
5820     };
5821
5822     return 1 if &$scan_config($conf);
5823
5824     undef $skip_drive;
5825
5826     foreach my $snapname (keys %{$conf->{snapshots}}) {
5827         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5828     }
5829
5830     return 0;
5831 }
5832
5833 sub update_disksize {
5834     my ($vmid, $conf, $volid_hash) = @_;
5835
5836     my $changes;
5837     my $prefix = "VM $vmid:";
5838
5839     # used and unused disks
5840     my $referenced = {};
5841
5842     # Note: it is allowed to define multiple storages with same path (alias), so
5843     # we need to check both 'volid' and real 'path' (two different volid can point
5844     # to the same path).
5845
5846     my $referencedpath = {};
5847
5848     # update size info
5849     foreach my $opt (keys %$conf) {
5850         if (is_valid_drivename($opt)) {
5851             my $drive = parse_drive($opt, $conf->{$opt});
5852             my $volid = $drive->{file};
5853             next if !$volid;
5854
5855             $referenced->{$volid} = 1;
5856             if ($volid_hash->{$volid} &&
5857                 (my $path = $volid_hash->{$volid}->{path})) {
5858                 $referencedpath->{$path} = 1;
5859             }
5860
5861             next if drive_is_cdrom($drive);
5862             next if !$volid_hash->{$volid};
5863
5864             $drive->{size} = $volid_hash->{$volid}->{size};
5865             my $new = print_drive($vmid, $drive);
5866             if ($new ne $conf->{$opt}) {
5867                 $changes = 1;
5868                 $conf->{$opt} = $new;
5869                 print "$prefix update disk '$opt' information.\n";
5870             }
5871         }
5872     }
5873
5874     # remove 'unusedX' entry if volume is used
5875     foreach my $opt (keys %$conf) {
5876         next if $opt !~ m/^unused\d+$/;
5877         my $volid = $conf->{$opt};
5878         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5879         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5880             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5881             $changes = 1;
5882             delete $conf->{$opt};
5883         }
5884
5885         $referenced->{$volid} = 1;
5886         $referencedpath->{$path} = 1 if $path;
5887     }
5888
5889     foreach my $volid (sort keys %$volid_hash) {
5890         next if $volid =~ m/vm-$vmid-state-/;
5891         next if $referenced->{$volid};
5892         my $path = $volid_hash->{$volid}->{path};
5893         next if !$path; # just to be sure
5894         next if $referencedpath->{$path};
5895         $changes = 1;
5896         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5897         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5898         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5899     }
5900
5901     return $changes;
5902 }
5903
5904 sub rescan {
5905     my ($vmid, $nolock, $dryrun) = @_;
5906
5907     my $cfg = PVE::Storage::config();
5908
5909     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5910     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5911     foreach my $stor (keys %{$cfg->{ids}}) {
5912         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5913     }
5914
5915     print "rescan volumes...\n";
5916     my $volid_hash = scan_volids($cfg, $vmid);
5917
5918     my $updatefn =  sub {
5919         my ($vmid) = @_;
5920
5921         my $conf = PVE::QemuConfig->load_config($vmid);
5922
5923         PVE::QemuConfig->check_lock($conf);
5924
5925         my $vm_volids = {};
5926         foreach my $volid (keys %$volid_hash) {
5927             my $info = $volid_hash->{$volid};
5928             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5929         }
5930
5931         my $changes = update_disksize($vmid, $conf, $vm_volids);
5932
5933         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5934     };
5935
5936     if (defined($vmid)) {
5937         if ($nolock) {
5938             &$updatefn($vmid);
5939         } else {
5940             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5941         }
5942     } else {
5943         my $vmlist = config_list();
5944         foreach my $vmid (keys %$vmlist) {
5945             if ($nolock) {
5946                 &$updatefn($vmid);
5947             } else {
5948                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5949             }
5950         }
5951     }
5952 }
5953
5954 sub restore_vma_archive {
5955     my ($archive, $vmid, $user, $opts, $comp) = @_;
5956
5957     my $readfrom = $archive;
5958
5959     my $cfg = PVE::Storage::config();
5960     my $commands = [];
5961     my $bwlimit = $opts->{bwlimit};
5962
5963     my $dbg_cmdstring = '';
5964     my $add_pipe = sub {
5965         my ($cmd) = @_;
5966         push @$commands, $cmd;
5967         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5968         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5969         $readfrom = '-';
5970     };
5971
5972     my $input = undef;
5973     if ($archive eq '-') {
5974         $input = '<&STDIN';
5975     } else {
5976         # If we use a backup from a PVE defined storage we also consider that
5977         # storage's rate limit:
5978         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5979         if (defined($volid)) {
5980             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5981             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5982             if ($readlimit) {
5983                 print STDERR "applying read rate limit: $readlimit\n";
5984                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5985                 $add_pipe->($cstream);
5986             }
5987         }
5988     }
5989
5990     if ($comp) {
5991         my $cmd;
5992         if ($comp eq 'gzip') {
5993             $cmd = ['zcat', $readfrom];
5994         } elsif ($comp eq 'lzop') {
5995             $cmd = ['lzop', '-d', '-c', $readfrom];
5996         } else {
5997             die "unknown compression method '$comp'\n";
5998         }
5999         $add_pipe->($cmd);
6000     }
6001
6002     my $tmpdir = "/var/tmp/vzdumptmp$$";
6003     rmtree $tmpdir;
6004
6005     # disable interrupts (always do cleanups)
6006     local $SIG{INT} =
6007         local $SIG{TERM} =
6008         local $SIG{QUIT} =
6009         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6010
6011     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6012     POSIX::mkfifo($mapfifo, 0600);
6013     my $fifofh;
6014
6015     my $openfifo = sub {
6016         open($fifofh, '>', $mapfifo) || die $!;
6017     };
6018
6019     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6020
6021     my $oldtimeout;
6022     my $timeout = 5;
6023
6024     my $devinfo = {};
6025
6026     my $rpcenv = PVE::RPCEnvironment::get();
6027
6028     my $conffile = PVE::QemuConfig->config_file($vmid);
6029     my $tmpfn = "$conffile.$$.tmp";
6030
6031     # Note: $oldconf is undef if VM does not exists
6032     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6033     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6034
6035     my %storage_limits;
6036
6037     my $print_devmap = sub {
6038         my $virtdev_hash = {};
6039
6040         my $cfgfn = "$tmpdir/qemu-server.conf";
6041
6042         # we can read the config - that is already extracted
6043         my $fh = IO::File->new($cfgfn, "r") ||
6044             "unable to read qemu-server.conf - $!\n";
6045
6046         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6047         if (-f $fwcfgfn) {
6048             my $pve_firewall_dir = '/etc/pve/firewall';
6049             mkdir $pve_firewall_dir; # make sure the dir exists
6050             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6051         }
6052
6053         while (defined(my $line = <$fh>)) {
6054             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6055                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6056                 die "archive does not contain data for drive '$virtdev'\n"
6057                     if !$devinfo->{$devname};
6058                 if (defined($opts->{storage})) {
6059                     $storeid = $opts->{storage} || 'local';
6060                 } elsif (!$storeid) {
6061                     $storeid = 'local';
6062                 }
6063                 $format = 'raw' if !$format;
6064                 $devinfo->{$devname}->{devname} = $devname;
6065                 $devinfo->{$devname}->{virtdev} = $virtdev;
6066                 $devinfo->{$devname}->{format} = $format;
6067                 $devinfo->{$devname}->{storeid} = $storeid;
6068
6069                 # check permission on storage
6070                 my $pool = $opts->{pool}; # todo: do we need that?
6071                 if ($user ne 'root@pam') {
6072                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6073                 }
6074
6075                 $storage_limits{$storeid} = $bwlimit;
6076
6077                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6078             }
6079         }
6080
6081         foreach my $key (keys %storage_limits) {
6082             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6083             next if !$limit;
6084             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6085             $storage_limits{$key} = $limit * 1024;
6086         }
6087
6088         foreach my $devname (keys %$devinfo) {
6089             die "found no device mapping information for device '$devname'\n"
6090                 if !$devinfo->{$devname}->{virtdev};
6091         }
6092
6093         # create empty/temp config
6094         if ($oldconf) {
6095             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6096             foreach_drive($oldconf, sub {
6097                 my ($ds, $drive) = @_;
6098
6099                 return if drive_is_cdrom($drive);
6100
6101                 my $volid = $drive->{file};
6102
6103                 return if !$volid || $volid =~ m|^/|;
6104
6105                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6106                 return if !$path || !$owner || ($owner != $vmid);
6107
6108                 # Note: only delete disk we want to restore
6109                 # other volumes will become unused
6110                 if ($virtdev_hash->{$ds}) {
6111                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6112                     if (my $err = $@) {
6113                         warn $err;
6114                     }
6115                 }
6116             });
6117
6118             # delete vmstate files
6119             # since after the restore we have no snapshots anymore
6120             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6121                 my $snap = $oldconf->{snapshots}->{$snapname};
6122                 if ($snap->{vmstate}) {
6123                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6124                     if (my $err = $@) {
6125                         warn $err;
6126                     }
6127                 }
6128             }
6129         }
6130
6131         my $map = {};
6132         foreach my $virtdev (sort keys %$virtdev_hash) {
6133             my $d = $virtdev_hash->{$virtdev};
6134             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6135             my $storeid = $d->{storeid};
6136             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6137
6138             my $map_opts = '';
6139             if (my $limit = $storage_limits{$storeid}) {
6140                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6141             }
6142
6143             # test if requested format is supported
6144             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6145             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6146             $d->{format} = $defFormat if !$supported;
6147
6148             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6149                                                   $d->{format}, undef, $alloc_size);
6150             print STDERR "new volume ID is '$volid'\n";
6151             $d->{volid} = $volid;
6152             my $path = PVE::Storage::path($cfg, $volid);
6153
6154             PVE::Storage::activate_volumes($cfg,[$volid]);
6155
6156             my $write_zeros = 1;
6157             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6158                 $write_zeros = 0;
6159             }
6160
6161             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6162
6163             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6164             $map->{$virtdev} = $volid;
6165         }
6166
6167         $fh->seek(0, 0) || die "seek failed - $!\n";
6168
6169         my $outfd = new IO::File ($tmpfn, "w") ||
6170             die "unable to write config for VM $vmid\n";
6171
6172         my $cookie = { netcount => 0 };
6173         while (defined(my $line = <$fh>)) {
6174             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6175         }
6176
6177         $fh->close();
6178         $outfd->close();
6179     };
6180
6181     eval {
6182         # enable interrupts
6183         local $SIG{INT} =
6184             local $SIG{TERM} =
6185             local $SIG{QUIT} =
6186             local $SIG{HUP} =
6187             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6188         local $SIG{ALRM} = sub { die "got timeout\n"; };
6189
6190         $oldtimeout = alarm($timeout);
6191
6192         my $parser = sub {
6193             my $line = shift;
6194
6195             print "$line\n";
6196
6197             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6198                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6199                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6200             } elsif ($line =~ m/^CTIME: /) {
6201                 # we correctly received the vma config, so we can disable
6202                 # the timeout now for disk allocation (set to 10 minutes, so
6203                 # that we always timeout if something goes wrong)
6204                 alarm(600);
6205                 &$print_devmap();
6206                 print $fifofh "done\n";
6207                 my $tmp = $oldtimeout || 0;
6208                 $oldtimeout = undef;
6209                 alarm($tmp);
6210                 close($fifofh);
6211             }
6212         };
6213
6214         print "restore vma archive: $dbg_cmdstring\n";
6215         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6216     };
6217     my $err = $@;
6218
6219     alarm($oldtimeout) if $oldtimeout;
6220
6221     my $vollist = [];
6222     foreach my $devname (keys %$devinfo) {
6223         my $volid = $devinfo->{$devname}->{volid};
6224         push @$vollist, $volid if $volid;
6225     }
6226
6227     PVE::Storage::deactivate_volumes($cfg, $vollist);
6228
6229     unlink $mapfifo;
6230
6231     if ($err) {
6232         rmtree $tmpdir;
6233         unlink $tmpfn;
6234
6235         foreach my $devname (keys %$devinfo) {
6236             my $volid = $devinfo->{$devname}->{volid};
6237             next if !$volid;
6238             eval {
6239                 if ($volid =~ m|^/|) {
6240                     unlink $volid || die 'unlink failed\n';
6241                 } else {
6242                     PVE::Storage::vdisk_free($cfg, $volid);
6243                 }
6244                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6245             };
6246             print STDERR "unable to cleanup '$volid' - $@" if $@;
6247         }
6248         die $err;
6249     }
6250
6251     rmtree $tmpdir;
6252
6253     rename($tmpfn, $conffile) ||
6254         die "unable to commit configuration file '$conffile'\n";
6255
6256     PVE::Cluster::cfs_update(); # make sure we read new file
6257
6258     eval { rescan($vmid, 1); };
6259     warn $@ if $@;
6260 }
6261
6262 sub restore_tar_archive {
6263     my ($archive, $vmid, $user, $opts) = @_;
6264
6265     if ($archive ne '-') {
6266         my $firstfile = tar_archive_read_firstfile($archive);
6267         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6268             if $firstfile ne 'qemu-server.conf';
6269     }
6270
6271     my $storecfg = PVE::Storage::config();
6272
6273     # destroy existing data - keep empty config
6274     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6275     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6276
6277     my $tocmd = "/usr/lib/qemu-server/qmextract";
6278
6279     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6280     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6281     $tocmd .= ' --prealloc' if $opts->{prealloc};
6282     $tocmd .= ' --info' if $opts->{info};
6283
6284     # tar option "xf" does not autodetect compression when read from STDIN,
6285     # so we pipe to zcat
6286     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6287         PVE::Tools::shellquote("--to-command=$tocmd");
6288
6289     my $tmpdir = "/var/tmp/vzdumptmp$$";
6290     mkpath $tmpdir;
6291
6292     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6293     local $ENV{VZDUMP_VMID} = $vmid;
6294     local $ENV{VZDUMP_USER} = $user;
6295
6296     my $conffile = PVE::QemuConfig->config_file($vmid);
6297     my $tmpfn = "$conffile.$$.tmp";
6298
6299     # disable interrupts (always do cleanups)
6300     local $SIG{INT} =
6301         local $SIG{TERM} =
6302         local $SIG{QUIT} =
6303         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6304
6305     eval {
6306         # enable interrupts
6307         local $SIG{INT} =
6308             local $SIG{TERM} =
6309             local $SIG{QUIT} =
6310             local $SIG{HUP} =
6311             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6312
6313         if ($archive eq '-') {
6314             print "extracting archive from STDIN\n";
6315             run_command($cmd, input => "<&STDIN");
6316         } else {
6317             print "extracting archive '$archive'\n";
6318             run_command($cmd);
6319         }
6320
6321         return if $opts->{info};
6322
6323         # read new mapping
6324         my $map = {};
6325         my $statfile = "$tmpdir/qmrestore.stat";
6326         if (my $fd = IO::File->new($statfile, "r")) {
6327             while (defined (my $line = <$fd>)) {
6328                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6329                     $map->{$1} = $2 if $1;
6330                 } else {
6331                     print STDERR "unable to parse line in statfile - $line\n";
6332                 }
6333             }
6334             $fd->close();
6335         }
6336
6337         my $confsrc = "$tmpdir/qemu-server.conf";
6338
6339         my $srcfd = new IO::File($confsrc, "r") ||
6340             die "unable to open file '$confsrc'\n";
6341
6342         my $outfd = new IO::File ($tmpfn, "w") ||
6343             die "unable to write config for VM $vmid\n";
6344
6345         my $cookie = { netcount => 0 };
6346         while (defined (my $line = <$srcfd>)) {
6347             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6348         }
6349
6350         $srcfd->close();
6351         $outfd->close();
6352     };
6353     my $err = $@;
6354
6355     if ($err) {
6356
6357         unlink $tmpfn;
6358
6359         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6360
6361         die $err;
6362     }
6363
6364     rmtree $tmpdir;
6365
6366     rename $tmpfn, $conffile ||
6367         die "unable to commit configuration file '$conffile'\n";
6368
6369     PVE::Cluster::cfs_update(); # make sure we read new file
6370
6371     eval { rescan($vmid, 1); };
6372     warn $@ if $@;
6373 };
6374
6375 sub foreach_storage_used_by_vm {
6376     my ($conf, $func) = @_;
6377
6378     my $sidhash = {};
6379
6380     foreach_drive($conf, sub {
6381         my ($ds, $drive) = @_;
6382         return if drive_is_cdrom($drive);
6383
6384         my $volid = $drive->{file};
6385
6386         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6387         $sidhash->{$sid} = $sid if $sid;
6388     });
6389
6390     foreach my $sid (sort keys %$sidhash) {
6391         &$func($sid);
6392     }
6393 }
6394
6395 sub do_snapshots_with_qemu {
6396     my ($storecfg, $volid) = @_;
6397
6398     my $storage_name = PVE::Storage::parse_volume_id($volid);
6399
6400     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6401         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6402         return 1;
6403     }
6404
6405     if ($volid =~ m/\.(qcow2|qed)$/){
6406         return 1;
6407     }
6408
6409     return undef;
6410 }
6411
6412 sub qga_check_running {
6413     my ($vmid, $nowarn) = @_;
6414
6415     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6416     if ($@) {
6417         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6418         return 0;
6419     }
6420     return 1;
6421 }
6422
6423 sub template_create {
6424     my ($vmid, $conf, $disk) = @_;
6425
6426     my $storecfg = PVE::Storage::config();
6427
6428     foreach_drive($conf, sub {
6429         my ($ds, $drive) = @_;
6430
6431         return if drive_is_cdrom($drive);
6432         return if $disk && $ds ne $disk;
6433
6434         my $volid = $drive->{file};
6435         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6436
6437         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6438         $drive->{file} = $voliddst;
6439         $conf->{$ds} = print_drive($vmid, $drive);
6440         PVE::QemuConfig->write_config($vmid, $conf);
6441     });
6442 }
6443
6444 sub qemu_img_convert {
6445     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6446
6447     my $storecfg = PVE::Storage::config();
6448     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6449     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6450
6451     if ($src_storeid && $dst_storeid) {
6452
6453         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6454
6455         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6456         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6457
6458         my $src_format = qemu_img_format($src_scfg, $src_volname);
6459         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6460
6461         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6462         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6463
6464         my $cmd = [];
6465         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6466         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6467         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6468         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6469         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6470         if ($is_zero_initialized) {
6471             push @$cmd, "zeroinit:$dst_path";
6472         } else {
6473             push @$cmd, $dst_path;
6474         }
6475
6476         my $parser = sub {
6477             my $line = shift;
6478             if($line =~ m/\((\S+)\/100\%\)/){
6479                 my $percent = $1;
6480                 my $transferred = int($size * $percent / 100);
6481                 my $remaining = $size - $transferred;
6482
6483                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6484             }
6485
6486         };
6487
6488         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6489         my $err = $@;
6490         die "copy failed: $err" if $err;
6491     }
6492 }
6493
6494 sub qemu_img_format {
6495     my ($scfg, $volname) = @_;
6496
6497     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6498         return $1;
6499     } else {
6500         return "raw";
6501     }
6502 }
6503
6504 sub qemu_drive_mirror {
6505     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6506
6507     $jobs = {} if !$jobs;
6508
6509     my $qemu_target;
6510     my $format;
6511     $jobs->{"drive-$drive"} = {};
6512
6513     if ($dst_volid =~ /^nbd:/) {
6514         $qemu_target = $dst_volid;
6515         $format = "nbd";
6516     } else {
6517         my $storecfg = PVE::Storage::config();
6518         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6519
6520         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6521
6522         $format = qemu_img_format($dst_scfg, $dst_volname);
6523
6524         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6525
6526         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6527     }
6528
6529     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6530     $opts->{format} = $format if $format;
6531
6532     print "drive mirror is starting for drive-$drive\n";
6533
6534     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6535
6536     if (my $err = $@) {
6537         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6538         die "mirroring error: $err";
6539     }
6540
6541     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6542 }
6543
6544 sub qemu_drive_mirror_monitor {
6545     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6546
6547     eval {
6548         my $err_complete = 0;
6549
6550         while (1) {
6551             die "storage migration timed out\n" if $err_complete > 300;
6552
6553             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6554
6555             my $running_mirror_jobs = {};
6556             foreach my $stat (@$stats) {
6557                 next if $stat->{type} ne 'mirror';
6558                 $running_mirror_jobs->{$stat->{device}} = $stat;
6559             }
6560
6561             my $readycounter = 0;
6562
6563             foreach my $job (keys %$jobs) {
6564
6565                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6566                     print "$job : finished\n";
6567                     delete $jobs->{$job};
6568                     next;
6569                 }
6570
6571                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6572
6573                 my $busy = $running_mirror_jobs->{$job}->{busy};
6574                 my $ready = $running_mirror_jobs->{$job}->{ready};
6575                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6576                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6577                     my $remaining = $total - $transferred;
6578                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6579
6580                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6581                 }
6582
6583                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6584             }
6585
6586             last if scalar(keys %$jobs) == 0;
6587
6588             if ($readycounter == scalar(keys %$jobs)) {
6589                 print "all mirroring jobs are ready \n";
6590                 last if $skipcomplete; #do the complete later
6591
6592                 if ($vmiddst && $vmiddst != $vmid) {
6593                     my $agent_running = $qga && qga_check_running($vmid);
6594                     if ($agent_running) {
6595                         print "freeze filesystem\n";
6596                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6597                     } else {
6598                         print "suspend vm\n";
6599                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6600                     }
6601
6602                     # if we clone a disk for a new target vm, we don't switch the disk
6603                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6604
6605                     if ($agent_running) {
6606                         print "unfreeze filesystem\n";
6607                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6608                     } else {
6609                         print "resume vm\n";
6610                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6611                     }
6612
6613                     last;
6614                 } else {
6615
6616                     foreach my $job (keys %$jobs) {
6617                         # try to switch the disk if source and destination are on the same guest
6618                         print "$job: Completing block job...\n";
6619
6620                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6621                         if ($@ =~ m/cannot be completed/) {
6622                             print "$job: Block job cannot be completed, try again.\n";
6623                             $err_complete++;
6624                         }else {
6625                             print "$job: Completed successfully.\n";
6626                             $jobs->{$job}->{complete} = 1;
6627                         }
6628                     }
6629                 }
6630             }
6631             sleep 1;
6632         }
6633     };
6634     my $err = $@;
6635
6636     if ($err) {
6637         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6638         die "mirroring error: $err";
6639     }
6640
6641 }
6642
6643 sub qemu_blockjobs_cancel {
6644     my ($vmid, $jobs) = @_;
6645
6646     foreach my $job (keys %$jobs) {
6647         print "$job: Cancelling block job\n";
6648         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6649         $jobs->{$job}->{cancel} = 1;
6650     }
6651
6652     while (1) {
6653         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6654
6655         my $running_jobs = {};
6656         foreach my $stat (@$stats) {
6657             $running_jobs->{$stat->{device}} = $stat;
6658         }
6659
6660         foreach my $job (keys %$jobs) {
6661
6662             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6663                 print "$job: Done.\n";
6664                 delete $jobs->{$job};
6665             }
6666         }
6667
6668         last if scalar(keys %$jobs) == 0;
6669
6670         sleep 1;
6671     }
6672 }
6673
6674 sub clone_disk {
6675     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6676         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6677
6678     my $newvolid;
6679
6680     if (!$full) {
6681         print "create linked clone of drive $drivename ($drive->{file})\n";
6682         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6683         push @$newvollist, $newvolid;
6684     } else {
6685
6686         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6687         $storeid = $storage if $storage;
6688
6689         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6690         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6691
6692         print "create full clone of drive $drivename ($drive->{file})\n";
6693         my $name = undef;
6694         if (drive_is_cloudinit($drive)) {
6695             $name = "vm-$newvmid-cloudinit";
6696             $snapname = undef;
6697             # cloudinit only supports raw and qcow2 atm:
6698             if ($dst_format eq 'qcow2') {
6699                 $name .= '.qcow2';
6700             } elsif ($dst_format ne 'raw') {
6701                 die "clone: unhandled format for cloudinit image\n";
6702             }
6703         }
6704         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6705         push @$newvollist, $newvolid;
6706
6707         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6708
6709         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6710         if (!$running || $snapname) {
6711             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6712         } else {
6713
6714             my $kvmver = get_running_qemu_version ($vmid);
6715             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6716                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6717                     if $drive->{iothread};
6718             }
6719
6720             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6721         }
6722     }
6723
6724     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6725
6726     my $disk = $drive;
6727     $disk->{format} = undef;
6728     $disk->{file} = $newvolid;
6729     $disk->{size} = $size;
6730
6731     return $disk;
6732 }
6733
6734 # this only works if VM is running
6735 sub get_current_qemu_machine {
6736     my ($vmid) = @_;
6737
6738     my $cmd = { execute => 'query-machines', arguments => {} };
6739     my $res = vm_qmp_command($vmid, $cmd);
6740
6741     my ($current, $default);
6742     foreach my $e (@$res) {
6743         $default = $e->{name} if $e->{'is-default'};
6744         $current = $e->{name} if $e->{'is-current'};
6745     }
6746
6747     # fallback to the default machine if current is not supported by qemu
6748     return $current || $default || 'pc';
6749 }
6750
6751 sub get_running_qemu_version {
6752     my ($vmid) = @_;
6753     my $cmd = { execute => 'query-version', arguments => {} };
6754     my $res = vm_qmp_command($vmid, $cmd);
6755     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6756 }
6757
6758 sub qemu_machine_feature_enabled {
6759     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6760
6761     my $current_major;
6762     my $current_minor;
6763
6764     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6765
6766         $current_major = $3;
6767         $current_minor = $4;
6768
6769     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6770
6771         $current_major = $1;
6772         $current_minor = $2;
6773     }
6774
6775     return 1 if $current_major > $version_major ||
6776                 ($current_major == $version_major &&
6777                  $current_minor >= $version_minor);
6778 }
6779
6780 sub qemu_machine_pxe {
6781     my ($vmid, $conf, $machine) = @_;
6782
6783     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6784
6785     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6786         $machine .= '.pxe';
6787     }
6788
6789     return $machine;
6790 }
6791
6792 sub qemu_use_old_bios_files {
6793     my ($machine_type) = @_;
6794
6795     return if !$machine_type;
6796
6797     my $use_old_bios_files = undef;
6798
6799     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6800         $machine_type = $1;
6801         $use_old_bios_files = 1;
6802     } else {
6803         my $kvmver = kvm_user_version();
6804         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6805         # load new efi bios files on migration. So this hack is required to allow
6806         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6807         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6808         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6809     }
6810
6811     return ($use_old_bios_files, $machine_type);
6812 }
6813
6814 sub create_efidisk($$$$$) {
6815     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6816
6817     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6818     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6819
6820     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6821     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6822     PVE::Storage::activate_volumes($storecfg, [$volid]);
6823
6824     my $path = PVE::Storage::path($storecfg, $volid);
6825     eval {
6826         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6827     };
6828     die "Copying EFI vars image failed: $@" if $@;
6829
6830     return ($volid, $vars_size);
6831 }
6832
6833 sub vm_iothreads_list {
6834     my ($vmid) = @_;
6835
6836     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6837
6838     my $iothreads = {};
6839     foreach my $iothread (@$res) {
6840         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6841     }
6842
6843     return $iothreads;
6844 }
6845
6846 sub scsihw_infos {
6847     my ($conf, $drive) = @_;
6848
6849     my $maxdev = 0;
6850
6851     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6852         $maxdev = 7;
6853     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6854         $maxdev = 1;
6855     } else {
6856         $maxdev = 256;
6857     }
6858
6859     my $controller = int($drive->{index} / $maxdev);
6860     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6861
6862     return ($maxdev, $controller, $controller_prefix);
6863 }
6864
6865 sub add_hyperv_enlightenments {
6866     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
6867
6868     return if $winversion < 6;
6869     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6870
6871     if ($gpu_passthrough || defined($hv_vendor_id)) {
6872         $hv_vendor_id //= 'proxmox';
6873         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
6874     }
6875
6876     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6877         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6878         push @$cpuFlags , 'hv_vapic';
6879         push @$cpuFlags , 'hv_time';
6880     } else {
6881         push @$cpuFlags , 'hv_spinlocks=0xffff';
6882     }
6883
6884     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6885         push @$cpuFlags , 'hv_reset';
6886         push @$cpuFlags , 'hv_vpindex';
6887         push @$cpuFlags , 'hv_runtime';
6888     }
6889
6890     if ($winversion >= 7) {
6891         push @$cpuFlags , 'hv_relaxed';
6892
6893         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6894             push @$cpuFlags , 'hv_synic';
6895             push @$cpuFlags , 'hv_stimer';
6896         }
6897     }
6898 }
6899
6900 sub windows_version {
6901     my ($ostype) = @_;
6902
6903     return 0 if !$ostype;
6904
6905     my $winversion = 0;
6906
6907     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6908         $winversion = 5;
6909     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6910         $winversion = 6;
6911     } elsif ($ostype =~ m/^win(\d+)$/) {
6912         $winversion = $1;
6913     }
6914
6915     return $winversion;
6916 }
6917
6918 sub resolve_dst_disk_format {
6919         my ($storecfg, $storeid, $src_volname, $format) = @_;
6920         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6921
6922         if (!$format) {
6923             # if no target format is specified, use the source disk format as hint
6924             if ($src_volname) {
6925                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6926                 $format = qemu_img_format($scfg, $src_volname);
6927             } else {
6928                 return $defFormat;
6929             }
6930         }
6931
6932         # test if requested format is supported - else use default
6933         my $supported = grep { $_ eq $format } @$validFormats;
6934         $format = $defFormat if !$supported;
6935         return $format;
6936 }
6937
6938 sub resolve_first_disk {
6939     my $conf = shift;
6940     my @disks = PVE::QemuServer::valid_drive_names();
6941     my $firstdisk;
6942     foreach my $ds (reverse @disks) {
6943         next if !$conf->{$ds};
6944         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6945         next if PVE::QemuServer::drive_is_cdrom($disk);
6946         $firstdisk = $ds;
6947     }
6948     return $firstdisk;
6949 }
6950
6951 sub generate_uuid {
6952     my ($uuid, $uuid_str);
6953     UUID::generate($uuid);
6954     UUID::unparse($uuid, $uuid_str);
6955     return $uuid_str;
6956 }
6957
6958 sub generate_smbios1_uuid {
6959     return "uuid=".generate_uuid();
6960 }
6961
6962 sub nbd_stop {
6963     my ($vmid) = @_;
6964
6965     vm_mon_cmd($vmid, 'nbd-server-stop');
6966 }
6967
6968 # bash completion helper
6969
6970 sub complete_backup_archives {
6971     my ($cmdname, $pname, $cvalue) = @_;
6972
6973     my $cfg = PVE::Storage::config();
6974
6975     my $storeid;
6976
6977     if ($cvalue =~ m/^([^:]+):/) {
6978         $storeid = $1;
6979     }
6980
6981     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6982
6983     my $res = [];
6984     foreach my $id (keys %$data) {
6985         foreach my $item (@{$data->{$id}}) {
6986             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6987             push @$res, $item->{volid} if defined($item->{volid});
6988         }
6989     }
6990
6991     return $res;
6992 }
6993
6994 my $complete_vmid_full = sub {
6995     my ($running) = @_;
6996
6997     my $idlist = vmstatus();
6998
6999     my $res = [];
7000
7001     foreach my $id (keys %$idlist) {
7002         my $d = $idlist->{$id};
7003         if (defined($running)) {
7004             next if $d->{template};
7005             next if $running && $d->{status} ne 'running';
7006             next if !$running && $d->{status} eq 'running';
7007         }
7008         push @$res, $id;
7009
7010     }
7011     return $res;
7012 };
7013
7014 sub complete_vmid {
7015     return &$complete_vmid_full();
7016 }
7017
7018 sub complete_vmid_stopped {
7019     return &$complete_vmid_full(0);
7020 }
7021
7022 sub complete_vmid_running {
7023     return &$complete_vmid_full(1);
7024 }
7025
7026 sub complete_storage {
7027
7028     my $cfg = PVE::Storage::config();
7029     my $ids = $cfg->{ids};
7030
7031     my $res = [];
7032     foreach my $sid (keys %$ids) {
7033         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7034         next if !$ids->{$sid}->{content}->{images};
7035         push @$res, $sid;
7036     }
7037
7038     return $res;
7039 }
7040
7041 1;