PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use Time::HiRes qw(gettimeofday);
  40 use File::Copy qw(copy);
  41 use URI::Escape;
  42
  43 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  44 my $OVMF = {
  45     x86_64 => [
  46         "$EDK2_FW_BASE/OVMF_CODE.fd",
  47         "$EDK2_FW_BASE/OVMF_VARS.fd"
  48     ],
  49     aarch64 => [
  50         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  51         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  52     ],
  53 };
  54
  55 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  56
  57 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  58
  59 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  60
  61 # Note about locking: we use flock on the config file protect
  62 # against concurent actions.
  63 # Aditionaly, we have a 'lock' setting in the config file. This
  64 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  65 # allowed when such lock is set. But you can ignore this kind of
  66 # lock with the --skiplock flag.
  67
  68 cfs_register_file('/qemu-server/',
  69                   \&parse_vm_config,
  70                   \&write_vm_config);
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  73     description => "Some command save/restore state from this location.",
  74     type => 'string',
  75     maxLength => 128,
  76     optional => 1,
  77 });
  78
  79 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  80     description => "The name of the snapshot.",
  81     type => 'string', format => 'pve-configid',
  82     maxLength => 40,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $cpu_vendor_list = {
 123     # Intel CPUs
 124     486 => 'GenuineIntel',
 125     pentium => 'GenuineIntel',
 126     pentium2  => 'GenuineIntel',
 127     pentium3  => 'GenuineIntel',
 128     coreduo => 'GenuineIntel',
 129     core2duo => 'GenuineIntel',
 130     Conroe  => 'GenuineIntel',
 131     Penryn  => 'GenuineIntel',
 132     Nehalem  => 'GenuineIntel',
 133     'Nehalem-IBRS'  => 'GenuineIntel',
 134     Westmere => 'GenuineIntel',
 135     'Westmere-IBRS' => 'GenuineIntel',
 136     SandyBridge => 'GenuineIntel',
 137     'SandyBridge-IBRS' => 'GenuineIntel',
 138     IvyBridge => 'GenuineIntel',
 139     'IvyBridge-IBRS' => 'GenuineIntel',
 140     Haswell => 'GenuineIntel',
 141     'Haswell-IBRS' => 'GenuineIntel',
 142     'Haswell-noTSX' => 'GenuineIntel',
 143     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 144     Broadwell => 'GenuineIntel',
 145     'Broadwell-IBRS' => 'GenuineIntel',
 146     'Broadwell-noTSX' => 'GenuineIntel',
 147     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 148     'Skylake-Client' => 'GenuineIntel',
 149     'Skylake-Client-IBRS' => 'GenuineIntel',
 150     'Skylake-Server' => 'GenuineIntel',
 151     'Skylake-Server-IBRS' => 'GenuineIntel',
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 174
 175 my $cpu_fmt = {
 176     cputype => {
 177         description => "Emulated CPU type.",
 178         type => 'string',
 179         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 180         default => 'kvm64',
 181         default_key => 1,
 182     },
 183     hidden => {
 184         description => "Do not identify as a KVM virtual machine.",
 185         type => 'boolean',
 186         optional => 1,
 187         default => 0
 188     },
 189     flags => {
 190         description => "List of additional CPU flags separated by ';'."
 191                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 192                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 193         format_description => '+FLAG[;-FLAG...]',
 194         type => 'string',
 195         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 196         optional => 1,
 197     },
 198 };
 199
 200 my $watchdog_fmt = {
 201     model => {
 202         default_key => 1,
 203         type => 'string',
 204         enum => [qw(i6300esb ib700)],
 205         description => "Watchdog type to emulate.",
 206         default => 'i6300esb',
 207         optional => 1,
 208     },
 209     action => {
 210         type => 'string',
 211         enum => [qw(reset shutdown poweroff pause debug none)],
 212         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 213         optional => 1,
 214     },
 215 };
 216 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 217
 218 my $agent_fmt = {
 219     enabled => {
 220         description => "Enable/disable Qemu GuestAgent.",
 221         type => 'boolean',
 222         default => 0,
 223         default_key => 1,
 224     },
 225     fstrim_cloned_disks => {
 226         description => "Run fstrim after cloning/moving a disk.",
 227         type => 'boolean',
 228         optional => 1,
 229         default => 0
 230     },
 231 };
 232
 233 my $vga_fmt = {
 234     type => {
 235         description => "Select the VGA type.",
 236         type => 'string',
 237         default => 'std',
 238         optional => 1,
 239         default_key => 1,
 240         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 241     },
 242     memory => {
 243         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 244         type => 'integer',
 245         optional => 1,
 246         minimum => 4,
 247         maximum => 512,
 248     },
 249 };
 250
 251 my $confdesc = {
 252     onboot => {
 253         optional => 1,
 254         type => 'boolean',
 255         description => "Specifies whether a VM will be started during system bootup.",
 256         default => 0,
 257     },
 258     autostart => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Automatic restart after crash (currently ignored).",
 262         default => 0,
 263     },
 264     hotplug => {
 265         optional => 1,
 266         type => 'string', format => 'pve-hotplug-features',
 267         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 268         default => 'network,disk,usb',
 269     },
 270     reboot => {
 271         optional => 1,
 272         type => 'boolean',
 273         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 274         default => 1,
 275     },
 276     lock => {
 277         optional => 1,
 278         type => 'string',
 279         description => "Lock/unlock the VM.",
 280         enum => [qw(migrate backup snapshot rollback)],
 281     },
 282     cpulimit => {
 283         optional => 1,
 284         type => 'number',
 285         description => "Limit of CPU usage.",
 286         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 287         minimum => 0,
 288         maximum => 128,
 289         default => 0,
 290     },
 291     cpuunits => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "CPU weight for a VM.",
 295         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 296         minimum => 2,
 297         maximum => 262144,
 298         default => 1024,
 299     },
 300     memory => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 304         minimum => 16,
 305         default => 512,
 306     },
 307     balloon => {
 308         optional => 1,
 309         type => 'integer',
 310         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 311         minimum => 0,
 312     },
 313     shares => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 317         minimum => 0,
 318         maximum => 50000,
 319         default => 1000,
 320     },
 321     keyboard => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 325                        "It should not be necessary to set it.",
 326         enum => PVE::Tools::kvmkeymaplist(),
 327         default => undef,
 328     },
 329     name => {
 330         optional => 1,
 331         type => 'string', format => 'dns-name',
 332         description => "Set a name for the VM. Only used on the configuration web interface.",
 333     },
 334     scsihw => {
 335         optional => 1,
 336         type => 'string',
 337         description => "SCSI controller model",
 338         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 339         default => 'lsi',
 340     },
 341     description => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 345     },
 346     ostype => {
 347         optional => 1,
 348         type => 'string',
 349         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 350         description => "Specify guest operating system.",
 351         verbose_description => <<EODESC,
 352 Specify guest operating system. This is used to enable special
 353 optimization/features for specific operating systems:
 354
 355 [horizontal]
 356 other;; unspecified OS
 357 wxp;; Microsoft Windows XP
 358 w2k;; Microsoft Windows 2000
 359 w2k3;; Microsoft Windows 2003
 360 w2k8;; Microsoft Windows 2008
 361 wvista;; Microsoft Windows Vista
 362 win7;; Microsoft Windows 7
 363 win8;; Microsoft Windows 8/2012/2012r2
 364 win10;; Microsoft Windows 10/2016
 365 l24;; Linux 2.4 Kernel
 366 l26;; Linux 2.6/3.X Kernel
 367 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 368 EODESC
 369     },
 370     boot => {
 371         optional => 1,
 372         type => 'string',
 373         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 374         pattern => '[acdn]{1,4}',
 375         default => 'cdn',
 376     },
 377     bootdisk => {
 378         optional => 1,
 379         type => 'string', format => 'pve-qm-bootdisk',
 380         description => "Enable booting from specified disk.",
 381         pattern => '(ide|sata|scsi|virtio)\d+',
 382     },
 383     smp => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPUs. Please use option -sockets instead.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     sockets => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of CPU sockets.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     cores => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "The number of cores per socket.",
 401         minimum => 1,
 402         default => 1,
 403     },
 404     numa => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Enable/disable NUMA.",
 408         default => 0,
 409     },
 410     hugepages => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Enable/disable hugepages memory.",
 414         enum => [qw(any 2 1024)],
 415     },
 416     vcpus => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "Number of hotplugged vcpus.",
 420         minimum => 1,
 421         default => 0,
 422     },
 423     acpi => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable ACPI.",
 427         default => 1,
 428     },
 429     agent => {
 430         optional => 1,
 431         description => "Enable/disable Qemu GuestAgent and its properties.",
 432         type => 'string',
 433         format => $agent_fmt,
 434     },
 435     kvm => {
 436         optional => 1,
 437         type => 'boolean',
 438         description => "Enable/disable KVM hardware virtualization.",
 439         default => 1,
 440     },
 441     tdf => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable time drift fix.",
 445         default => 0,
 446     },
 447     localtime => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 451     },
 452     freeze => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 456     },
 457     vga => {
 458         optional => 1,
 459         type => 'string', format => $vga_fmt,
 460         description => "Configure the VGA hardware.",
 461         verbose_description => "Configure the VGA Hardware. If you want to use ".
 462             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 463             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 464             "is 'std' for all OS types besides some Windows versions (XP and " .
 465             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 466             "display server. For win* OS you can select how many independent " .
 467             "displays you want, Linux guests can add displays them self.\n".
 468             "You can also run without any graphic card, using a serial device as terminal.",
 469     },
 470     watchdog => {
 471         optional => 1,
 472         type => 'string', format => 'pve-qm-watchdog',
 473         description => "Create a virtual hardware watchdog device.",
 474         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 475             " (by a guest action), the watchdog must be periodically polled " .
 476             "by an agent inside the guest or else the watchdog will reset " .
 477             "the guest (or execute the respective action specified)",
 478     },
 479     startdate => {
 480         optional => 1,
 481         type => 'string',
 482         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 483         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 484         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 485         default => 'now',
 486     },
 487     startup =>  get_standard_option('pve-startup-order'),
 488     template => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable Template.",
 492         default => 0,
 493     },
 494     args => {
 495         optional => 1,
 496         type => 'string',
 497         description => "Arbitrary arguments passed to kvm.",
 498         verbose_description => <<EODESCR,
 499 Arbitrary arguments passed to kvm, for example:
 500
 501 args: -no-reboot -no-hpet
 502
 503 NOTE: this option is for experts only.
 504 EODESCR
 505     },
 506     tablet => {
 507         optional => 1,
 508         type => 'boolean',
 509         default => 1,
 510         description => "Enable/disable the USB tablet device.",
 511         verbose_description => "Enable/disable the USB tablet device. This device is " .
 512             "usually needed to allow absolute mouse positioning with VNC. " .
 513             "Else the mouse runs out of sync with normal VNC clients. " .
 514             "If you're running lots of console-only guests on one host, " .
 515             "you may consider disabling this to save some context switches. " .
 516             "This is turned off by default if you use spice (-vga=qxl).",
 517     },
 518     migrate_speed => {
 519         optional => 1,
 520         type => 'integer',
 521         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 522         minimum => 0,
 523         default => 0,
 524     },
 525     migrate_downtime => {
 526         optional => 1,
 527         type => 'number',
 528         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 529         minimum => 0,
 530         default => 0.1,
 531     },
 532     cdrom => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-ide',
 535         typetext => '<volume>',
 536         description => "This is an alias for option -ide2",
 537     },
 538     cpu => {
 539         optional => 1,
 540         description => "Emulated CPU type.",
 541         type => 'string',
 542         format => $cpu_fmt,
 543     },
 544     parent => get_standard_option('pve-snapshot-name', {
 545         optional => 1,
 546         description => "Parent snapshot name. This is used internally, and should not be modified.",
 547     }),
 548     snaptime => {
 549         optional => 1,
 550         description => "Timestamp for snapshots.",
 551         type => 'integer',
 552         minimum => 0,
 553     },
 554     vmstate => {
 555         optional => 1,
 556         type => 'string', format => 'pve-volume-id',
 557         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 558     },
 559     vmstatestorage => get_standard_option('pve-storage-id', {
 560         description => "Default storage for VM state volumes/files.",
 561         optional => 1,
 562     }),
 563     runningmachine => get_standard_option('pve-qemu-machine', {
 564         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 565     }),
 566     machine => get_standard_option('pve-qemu-machine'),
 567     arch => {
 568         description => "Virtual processor architecture. Defaults to the host.",
 569         optional => 1,
 570         type => 'string',
 571         enum => [qw(x86_64 aarch64)],
 572     },
 573     smbios1 => {
 574         description => "Specify SMBIOS type 1 fields.",
 575         type => 'string', format => 'pve-qm-smbios1',
 576         maxLength => 256,
 577         optional => 1,
 578     },
 579     protection => {
 580         optional => 1,
 581         type => 'boolean',
 582         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 583         default => 0,
 584     },
 585     bios => {
 586         optional => 1,
 587         type => 'string',
 588         enum => [ qw(seabios ovmf) ],
 589         description => "Select BIOS implementation.",
 590         default => 'seabios',
 591     },
 592     vmgenid => {
 593         type => 'string',
 594         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 595         format_description => 'UUID',
 596         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 597         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 598             " 128-bit integer value identifier to the guest OS. This allows to".
 599             " notify the guest operating system when the virtual machine is".
 600             " executed with a different configuration (e.g. snapshot execution".
 601             " or creation from a template). The guest operating system notices".
 602             " the change, and is then able to react as appropriate by marking".
 603             " its copies of distributed databases as dirty, re-initializing its".
 604             " random number generator, etc.\n".
 605             "Note that auto-creation only works when done throug API/CLI create".
 606             " or update methods, but not when manually editing the config file.",
 607         default => "1 (autogenerated)",
 608         optional => 1,
 609     },
 610 };
 611
 612 my $confdesc_cloudinit = {
 613     citype => {
 614         optional => 1,
 615         type => 'string',
 616         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 617         enum => ['configdrive2', 'nocloud'],
 618     },
 619     ciuser => {
 620         optional => 1,
 621         type => 'string',
 622         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 623     },
 624     cipassword => {
 625         optional => 1,
 626         type => 'string',
 627         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 628     },
 629     searchdomain => {
 630         optional => 1,
 631         type => 'string',
 632         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 633     },
 634     nameserver => {
 635         optional => 1,
 636         type => 'string', format => 'address-list',
 637         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 638     },
 639     sshkeys => {
 640         optional => 1,
 641         type => 'string',
 642         format => 'urlencoded',
 643         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 644     },
 645 };
 646
 647 # what about other qemu settings ?
 648 #cpu => 'string',
 649 #machine => 'string',
 650 #fda => 'file',
 651 #fdb => 'file',
 652 #mtdblock => 'file',
 653 #sd => 'file',
 654 #pflash => 'file',
 655 #snapshot => 'bool',
 656 #bootp => 'file',
 657 ##tftp => 'dir',
 658 ##smb => 'dir',
 659 #kernel => 'file',
 660 #append => 'string',
 661 #initrd => 'file',
 662 ##soundhw => 'string',
 663
 664 while (my ($k, $v) = each %$confdesc) {
 665     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 666 }
 667
 668 my $MAX_IDE_DISKS = 4;
 669 my $MAX_SCSI_DISKS = 14;
 670 my $MAX_VIRTIO_DISKS = 16;
 671 my $MAX_SATA_DISKS = 6;
 672 my $MAX_USB_DEVICES = 5;
 673 my $MAX_NETS = 32;
 674 my $MAX_UNUSED_DISKS = 256;
 675 my $MAX_HOSTPCI_DEVICES = 4;
 676 my $MAX_SERIAL_PORTS = 4;
 677 my $MAX_PARALLEL_PORTS = 3;
 678 my $MAX_NUMA = 8;
 679
 680 my $numa_fmt = {
 681     cpus => {
 682         type => "string",
 683         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 684         description => "CPUs accessing this NUMA node.",
 685         format_description => "id[-id];...",
 686     },
 687     memory => {
 688         type => "number",
 689         description => "Amount of memory this NUMA node provides.",
 690         optional => 1,
 691     },
 692     hostnodes => {
 693         type => "string",
 694         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 695         description => "Host NUMA nodes to use.",
 696         format_description => "id[-id];...",
 697         optional => 1,
 698     },
 699     policy => {
 700         type => 'string',
 701         enum => [qw(preferred bind interleave)],
 702         description => "NUMA allocation policy.",
 703         optional => 1,
 704     },
 705 };
 706 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 707 my $numadesc = {
 708     optional => 1,
 709     type => 'string', format => $numa_fmt,
 710     description => "NUMA topology.",
 711 };
 712 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 713
 714 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 715     $confdesc->{"numa$i"} = $numadesc;
 716 }
 717
 718 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 719                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 720                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 721 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 722
 723 my $net_fmt_bridge_descr = <<__EOD__;
 724 Bridge to attach the network device to. The Proxmox VE standard bridge
 725 is called 'vmbr0'.
 726
 727 If you do not specify a bridge, we create a kvm user (NATed) network
 728 device, which provides DHCP and DNS services. The following addresses
 729 are used:
 730
 731  10.0.2.2   Gateway
 732  10.0.2.3   DNS Server
 733  10.0.2.4   SMB Server
 734
 735 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 736 __EOD__
 737
 738 my $net_fmt = {
 739     macaddr => {
 740         type => 'string',
 741         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 742         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 743         format_description => "XX:XX:XX:XX:XX:XX",
 744         optional => 1,
 745     },
 746     model => {
 747         type => 'string',
 748         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 749         enum => $nic_model_list,
 750         default_key => 1,
 751     },
 752     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 753     bridge => {
 754         type => 'string',
 755         description => $net_fmt_bridge_descr,
 756         format_description => 'bridge',
 757         optional => 1,
 758     },
 759     queues => {
 760         type => 'integer',
 761         minimum => 0, maximum => 16,
 762         description => 'Number of packet queues to be used on the device.',
 763         optional => 1,
 764     },
 765     rate => {
 766         type => 'number',
 767         minimum => 0,
 768         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 769         optional => 1,
 770     },
 771     tag => {
 772         type => 'integer',
 773         minimum => 1, maximum => 4094,
 774         description => 'VLAN tag to apply to packets on this interface.',
 775         optional => 1,
 776     },
 777     trunks => {
 778         type => 'string',
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => 'VLAN trunks to pass through this interface.',
 781         format_description => 'vlanid[;vlanid...]',
 782         optional => 1,
 783     },
 784     firewall => {
 785         type => 'boolean',
 786         description => 'Whether this interface should be protected by the firewall.',
 787         optional => 1,
 788     },
 789     link_down => {
 790         type => 'boolean',
 791         description => 'Whether this interface should be disconnected (like pulling the plug).',
 792         optional => 1,
 793     },
 794 };
 795
 796 my $netdesc = {
 797     optional => 1,
 798     type => 'string', format => $net_fmt,
 799     description => "Specify network devices.",
 800 };
 801
 802 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 803
 804 my $ipconfig_fmt = {
 805     ip => {
 806         type => 'string',
 807         format => 'pve-ipv4-config',
 808         format_description => 'IPv4Format/CIDR',
 809         description => 'IPv4 address in CIDR format.',
 810         optional => 1,
 811         default => 'dhcp',
 812     },
 813     gw => {
 814         type => 'string',
 815         format => 'ipv4',
 816         format_description => 'GatewayIPv4',
 817         description => 'Default gateway for IPv4 traffic.',
 818         optional => 1,
 819         requires => 'ip',
 820     },
 821     ip6 => {
 822         type => 'string',
 823         format => 'pve-ipv6-config',
 824         format_description => 'IPv6Format/CIDR',
 825         description => 'IPv6 address in CIDR format.',
 826         optional => 1,
 827         default => 'dhcp',
 828     },
 829     gw6 => {
 830         type => 'string',
 831         format => 'ipv6',
 832         format_description => 'GatewayIPv6',
 833         description => 'Default gateway for IPv6 traffic.',
 834         optional => 1,
 835         requires => 'ip6',
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 839 my $ipconfigdesc = {
 840     optional => 1,
 841     type => 'string', format => 'pve-qm-ipconfig',
 842     description => <<'EODESCR',
 843 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 844
 845 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 846
 847 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 848 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 849
 850 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 851 EODESCR
 852 };
 853 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 854
 855 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 856     $confdesc->{"net$i"} = $netdesc;
 857     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 858 }
 859
 860 foreach my $key (keys %$confdesc_cloudinit) {
 861     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 862 }
 863
 864 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 865 sub verify_volume_id_or_qm_path {
 866     my ($volid, $noerr) = @_;
 867
 868     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 869         return $volid;
 870     }
 871
 872     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 873     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 874     if ($@) {
 875         return undef if $noerr;
 876         die $@;
 877     }
 878     return $volid;
 879 }
 880
 881 my $drivename_hash;
 882
 883 my %drivedesc_base = (
 884     volume => { alias => 'file' },
 885     file => {
 886         type => 'string',
 887         format => 'pve-volume-id-or-qm-path',
 888         default_key => 1,
 889         format_description => 'volume',
 890         description => "The drive's backing volume.",
 891     },
 892     media => {
 893         type => 'string',
 894         enum => [qw(cdrom disk)],
 895         description => "The drive's media type.",
 896         default => 'disk',
 897         optional => 1
 898     },
 899     cyls => {
 900         type => 'integer',
 901         description => "Force the drive's physical geometry to have a specific cylinder count.",
 902         optional => 1
 903     },
 904     heads => {
 905         type => 'integer',
 906         description => "Force the drive's physical geometry to have a specific head count.",
 907         optional => 1
 908     },
 909     secs => {
 910         type => 'integer',
 911         description => "Force the drive's physical geometry to have a specific sector count.",
 912         optional => 1
 913     },
 914     trans => {
 915         type => 'string',
 916         enum => [qw(none lba auto)],
 917         description => "Force disk geometry bios translation mode.",
 918         optional => 1,
 919     },
 920     snapshot => {
 921         type => 'boolean',
 922         description => "Controls qemu's snapshot mode feature."
 923             . " If activated, changes made to the disk are temporary and will"
 924             . " be discarded when the VM is shutdown.",
 925         optional => 1,
 926     },
 927     cache => {
 928         type => 'string',
 929         enum => [qw(none writethrough writeback unsafe directsync)],
 930         description => "The drive's cache mode",
 931         optional => 1,
 932     },
 933     format => get_standard_option('pve-qm-image-format'),
 934     size => {
 935         type => 'string',
 936         format => 'disk-size',
 937         format_description => 'DiskSize',
 938         description => "Disk size. This is purely informational and has no effect.",
 939         optional => 1,
 940     },
 941     backup => {
 942         type => 'boolean',
 943         description => "Whether the drive should be included when making backups.",
 944         optional => 1,
 945     },
 946     replicate => {
 947         type => 'boolean',
 948         description => 'Whether the drive should considered for replication jobs.',
 949         optional => 1,
 950         default => 1,
 951     },
 952     rerror => {
 953         type => 'string',
 954         enum => [qw(ignore report stop)],
 955         description => 'Read error action.',
 956         optional => 1,
 957     },
 958     werror => {
 959         type => 'string',
 960         enum => [qw(enospc ignore report stop)],
 961         description => 'Write error action.',
 962         optional => 1,
 963     },
 964     aio => {
 965         type => 'string',
 966         enum => [qw(native threads)],
 967         description => 'AIO type to use.',
 968         optional => 1,
 969     },
 970     discard => {
 971         type => 'string',
 972         enum => [qw(ignore on)],
 973         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 974         optional => 1,
 975     },
 976     detect_zeroes => {
 977         type => 'boolean',
 978         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 979         optional => 1,
 980     },
 981     serial => {
 982         type => 'string',
 983         format => 'urlencoded',
 984         format_description => 'serial',
 985         maxLength => 20*3, # *3 since it's %xx url enoded
 986         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 987         optional => 1,
 988     },
 989     shared => {
 990         type => 'boolean',
 991         description => 'Mark this locally-managed volume as available on all nodes',
 992         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 993         optional => 1,
 994         default => 0,
 995     }
 996 );
 997
 998 my %iothread_fmt = ( iothread => {
 999         type => 'boolean',
1000         description => "Whether to use iothreads for this drive",
1001         optional => 1,
1002 });
1003
1004 my %model_fmt = (
1005     model => {
1006         type => 'string',
1007         format => 'urlencoded',
1008         format_description => 'model',
1009         maxLength => 40*3, # *3 since it's %xx url enoded
1010         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1011         optional => 1,
1012     },
1013 );
1014
1015 my %queues_fmt = (
1016     queues => {
1017         type => 'integer',
1018         description => "Number of queues.",
1019         minimum => 2,
1020         optional => 1
1021     }
1022 );
1023
1024 my %scsiblock_fmt = (
1025     scsiblock => {
1026         type => 'boolean',
1027         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1028         optional => 1,
1029         default => 0,
1030     },
1031 );
1032
1033 my %ssd_fmt = (
1034     ssd => {
1035         type => 'boolean',
1036         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1037         optional => 1,
1038     },
1039 );
1040
1041 my $add_throttle_desc = sub {
1042     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1043     my $d = {
1044         type => $type,
1045         format_description => $unit,
1046         description => "Maximum $what in $longunit.",
1047         optional => 1,
1048     };
1049     $d->{minimum} = $minimum if defined($minimum);
1050     $drivedesc_base{$key} = $d;
1051 };
1052 # throughput: (leaky bucket)
1053 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1054 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1056 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1057 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1059 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1060 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1061 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1062
1063 # pools: (pool of IO before throttling starts taking effect)
1064 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1065 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1067 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1068 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1069 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1070
1071 # burst lengths
1072 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1073 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1075 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1078
1079 # legacy support
1080 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1081 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1082 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1083 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1084
1085 my $ide_fmt = {
1086     %drivedesc_base,
1087     %model_fmt,
1088     %ssd_fmt,
1089 };
1090 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1091
1092 my $idedesc = {
1093     optional => 1,
1094     type => 'string', format => $ide_fmt,
1095     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1096 };
1097 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1098
1099 my $scsi_fmt = {
1100     %drivedesc_base,
1101     %iothread_fmt,
1102     %queues_fmt,
1103     %scsiblock_fmt,
1104     %ssd_fmt,
1105 };
1106 my $scsidesc = {
1107     optional => 1,
1108     type => 'string', format => $scsi_fmt,
1109     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1112
1113 my $sata_fmt = {
1114     %drivedesc_base,
1115     %ssd_fmt,
1116 };
1117 my $satadesc = {
1118     optional => 1,
1119     type => 'string', format => $sata_fmt,
1120     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1123
1124 my $virtio_fmt = {
1125     %drivedesc_base,
1126     %iothread_fmt,
1127 };
1128 my $virtiodesc = {
1129     optional => 1,
1130     type => 'string', format => $virtio_fmt,
1131     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1132 };
1133 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1134
1135 my $alldrive_fmt = {
1136     %drivedesc_base,
1137     %iothread_fmt,
1138     %model_fmt,
1139     %queues_fmt,
1140     %scsiblock_fmt,
1141     %ssd_fmt,
1142 };
1143
1144 my $efidisk_fmt = {
1145     volume => { alias => 'file' },
1146     file => {
1147         type => 'string',
1148         format => 'pve-volume-id-or-qm-path',
1149         default_key => 1,
1150         format_description => 'volume',
1151         description => "The drive's backing volume.",
1152     },
1153     format => get_standard_option('pve-qm-image-format'),
1154     size => {
1155         type => 'string',
1156         format => 'disk-size',
1157         format_description => 'DiskSize',
1158         description => "Disk size. This is purely informational and has no effect.",
1159         optional => 1,
1160     },
1161 };
1162
1163 my $efidisk_desc = {
1164     optional => 1,
1165     type => 'string', format => $efidisk_fmt,
1166     description => "Configure a Disk for storing EFI vars",
1167 };
1168
1169 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1170
1171 my $usb_fmt = {
1172     host => {
1173         default_key => 1,
1174         type => 'string', format => 'pve-qm-usb-device',
1175         format_description => 'HOSTUSBDEVICE|spice',
1176         description => <<EODESCR,
1177 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1178
1179  'bus-port(.port)*' (decimal numbers) or
1180  'vendor_id:product_id' (hexadeciaml numbers) or
1181  'spice'
1182
1183 You can use the 'lsusb -t' command to list existing usb devices.
1184
1185 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1186
1187 The value 'spice' can be used to add a usb redirection devices for spice.
1188 EODESCR
1189     },
1190     usb3 => {
1191         optional => 1,
1192         type => 'boolean',
1193         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1194         default => 0,
1195     },
1196 };
1197
1198 my $usbdesc = {
1199     optional => 1,
1200     type => 'string', format => $usb_fmt,
1201     description => "Configure an USB device (n is 0 to 4).",
1202 };
1203 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1204
1205 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1206 my $hostpci_fmt = {
1207     host => {
1208         default_key => 1,
1209         type => 'string',
1210         pattern => qr/$PCIRE(;$PCIRE)*/,
1211         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1212         description => <<EODESCR,
1213 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1214 of PCI virtual functions of the host. HOSTPCIID syntax is:
1215
1216 'bus:dev.func' (hexadecimal numbers)
1217
1218 You can us the 'lspci' command to list existing PCI devices.
1219 EODESCR
1220     },
1221     rombar => {
1222         type => 'boolean',
1223         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1224         optional => 1,
1225         default => 1,
1226     },
1227     romfile => {
1228         type => 'string',
1229         pattern => '[^,;]+',
1230         format_description => 'string',
1231         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1232         optional => 1,
1233     },
1234     pcie => {
1235         type => 'boolean',
1236         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1237         optional => 1,
1238         default => 0,
1239     },
1240     'x-vga' => {
1241         type => 'boolean',
1242         description =>  "Enable vfio-vga device support.",
1243         optional => 1,
1244         default => 0,
1245     },
1246     'mdev' => {
1247         type => 'string',
1248         format_description => 'string',
1249         pattern => '[^/\.:]+',
1250         optional => 1,
1251         description => <<EODESCR
1252 The type of mediated device to use.
1253 An instance of this type will be created on startup of the VM and
1254 will be cleaned up when the VM stops.
1255 EODESCR
1256     }
1257 };
1258 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1259
1260 my $hostpcidesc = {
1261         optional => 1,
1262         type => 'string', format => 'pve-qm-hostpci',
1263         description => "Map host PCI devices into guest.",
1264         verbose_description =>  <<EODESCR,
1265 Map host PCI devices into guest.
1266
1267 NOTE: This option allows direct access to host hardware. So it is no longer
1268 possible to migrate such machines - use with special care.
1269
1270 CAUTION: Experimental! User reported problems with this option.
1271 EODESCR
1272 };
1273 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1274
1275 my $serialdesc = {
1276         optional => 1,
1277         type => 'string',
1278         pattern => '(/dev/.+|socket)',
1279         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1280         verbose_description =>  <<EODESCR,
1281 Create a serial device inside the VM (n is 0 to 3), and pass through a
1282 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1283 host side (use 'qm terminal' to open a terminal connection).
1284
1285 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1286
1287 CAUTION: Experimental! User reported problems with this option.
1288 EODESCR
1289 };
1290
1291 my $paralleldesc= {
1292         optional => 1,
1293         type => 'string',
1294         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1295         description =>  "Map host parallel devices (n is 0 to 2).",
1296         verbose_description =>  <<EODESCR,
1297 Map host parallel devices (n is 0 to 2).
1298
1299 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1300
1301 CAUTION: Experimental! User reported problems with this option.
1302 EODESCR
1303 };
1304
1305 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1306     $confdesc->{"parallel$i"} = $paralleldesc;
1307 }
1308
1309 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1310     $confdesc->{"serial$i"} = $serialdesc;
1311 }
1312
1313 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1314     $confdesc->{"hostpci$i"} = $hostpcidesc;
1315 }
1316
1317 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1318     $drivename_hash->{"ide$i"} = 1;
1319     $confdesc->{"ide$i"} = $idedesc;
1320 }
1321
1322 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1323     $drivename_hash->{"sata$i"} = 1;
1324     $confdesc->{"sata$i"} = $satadesc;
1325 }
1326
1327 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1328     $drivename_hash->{"scsi$i"} = 1;
1329     $confdesc->{"scsi$i"} = $scsidesc ;
1330 }
1331
1332 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1333     $drivename_hash->{"virtio$i"} = 1;
1334     $confdesc->{"virtio$i"} = $virtiodesc;
1335 }
1336
1337 $drivename_hash->{efidisk0} = 1;
1338 $confdesc->{efidisk0} = $efidisk_desc;
1339
1340 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1341     $confdesc->{"usb$i"} = $usbdesc;
1342 }
1343
1344 my $unuseddesc = {
1345     optional => 1,
1346     type => 'string', format => 'pve-volume-id',
1347     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1348 };
1349
1350 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1351     $confdesc->{"unused$i"} = $unuseddesc;
1352 }
1353
1354 my $kvm_api_version = 0;
1355
1356 sub kvm_version {
1357     return $kvm_api_version if $kvm_api_version;
1358
1359     open my $fh, '<', '/dev/kvm'
1360         or return undef;
1361
1362     # 0xae00 => KVM_GET_API_VERSION
1363     $kvm_api_version = ioctl($fh, 0xae00, 0);
1364
1365     return $kvm_api_version;
1366 }
1367
1368 my $kvm_user_version;
1369
1370 sub kvm_user_version {
1371
1372     return $kvm_user_version if $kvm_user_version;
1373
1374     $kvm_user_version = 'unknown';
1375
1376     my $code = sub {
1377         my $line = shift;
1378         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1379             $kvm_user_version = $2;
1380         }
1381     };
1382
1383     eval { run_command("kvm -version", outfunc => $code); };
1384     warn $@ if $@;
1385
1386     return $kvm_user_version;
1387
1388 }
1389
1390 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1391
1392 sub valid_drive_names {
1393     # order is important - used to autoselect boot disk
1394     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1395             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1396             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1397             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1398             'efidisk0');
1399 }
1400
1401 sub is_valid_drivename {
1402     my $dev = shift;
1403
1404     return defined($drivename_hash->{$dev});
1405 }
1406
1407 sub option_exists {
1408     my $key = shift;
1409     return defined($confdesc->{$key});
1410 }
1411
1412 sub nic_models {
1413     return $nic_model_list;
1414 }
1415
1416 sub os_list_description {
1417
1418     return {
1419         other => 'Other',
1420         wxp => 'Windows XP',
1421         w2k => 'Windows 2000',
1422         w2k3 =>, 'Windows 2003',
1423         w2k8 => 'Windows 2008',
1424         wvista => 'Windows Vista',
1425         win7 => 'Windows 7',
1426         win8 => 'Windows 8/2012',
1427         win10 => 'Windows 10/2016',
1428         l24 => 'Linux 2.4',
1429         l26 => 'Linux 2.6',
1430     };
1431 }
1432
1433 my $cdrom_path;
1434
1435 sub get_cdrom_path {
1436
1437     return  $cdrom_path if $cdrom_path;
1438
1439     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1440     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1441     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1442 }
1443
1444 sub get_iso_path {
1445     my ($storecfg, $vmid, $cdrom) = @_;
1446
1447     if ($cdrom eq 'cdrom') {
1448         return get_cdrom_path();
1449     } elsif ($cdrom eq 'none') {
1450         return '';
1451     } elsif ($cdrom =~ m|^/|) {
1452         return $cdrom;
1453     } else {
1454         return PVE::Storage::path($storecfg, $cdrom);
1455     }
1456 }
1457
1458 # try to convert old style file names to volume IDs
1459 sub filename_to_volume_id {
1460     my ($vmid, $file, $media) = @_;
1461
1462      if (!($file eq 'none' || $file eq 'cdrom' ||
1463           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1464
1465         return undef if $file =~ m|/|;
1466
1467         if ($media && $media eq 'cdrom') {
1468             $file = "local:iso/$file";
1469         } else {
1470             $file = "local:$vmid/$file";
1471         }
1472     }
1473
1474     return $file;
1475 }
1476
1477 sub verify_media_type {
1478     my ($opt, $vtype, $media) = @_;
1479
1480     return if !$media;
1481
1482     my $etype;
1483     if ($media eq 'disk') {
1484         $etype = 'images';
1485     } elsif ($media eq 'cdrom') {
1486         $etype = 'iso';
1487     } else {
1488         die "internal error";
1489     }
1490
1491     return if ($vtype eq $etype);
1492
1493     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1494 }
1495
1496 sub cleanup_drive_path {
1497     my ($opt, $storecfg, $drive) = @_;
1498
1499     # try to convert filesystem paths to volume IDs
1500
1501     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1502         ($drive->{file} !~ m|^/dev/.+|) &&
1503         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1504         ($drive->{file} !~ m/^\d+$/)) {
1505         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1506         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1507         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1508         verify_media_type($opt, $vtype, $drive->{media});
1509         $drive->{file} = $volid;
1510     }
1511
1512     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1513 }
1514
1515 sub parse_hotplug_features {
1516     my ($data) = @_;
1517
1518     my $res = {};
1519
1520     return $res if $data eq '0';
1521
1522     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1523
1524     foreach my $feature (PVE::Tools::split_list($data)) {
1525         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1526             $res->{$1} = 1;
1527         } else {
1528             die "invalid hotplug feature '$feature'\n";
1529         }
1530     }
1531     return $res;
1532 }
1533
1534 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1535 sub pve_verify_hotplug_features {
1536     my ($value, $noerr) = @_;
1537
1538     return $value if parse_hotplug_features($value);
1539
1540     return undef if $noerr;
1541
1542     die "unable to parse hotplug option\n";
1543 }
1544
1545 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1546 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1547 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1548 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1549 #        [,iothread=on][,serial=serial][,model=model]
1550
1551 sub parse_drive {
1552     my ($key, $data) = @_;
1553
1554     my ($interface, $index);
1555
1556     if ($key =~ m/^([^\d]+)(\d+)$/) {
1557         $interface = $1;
1558         $index = $2;
1559     } else {
1560         return undef;
1561     }
1562
1563     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1564                                      : $confdesc->{$key}->{format};
1565     if (!$desc) {
1566         warn "invalid drive key: $key\n";
1567         return undef;
1568     }
1569     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1570     return undef if !$res;
1571     $res->{interface} = $interface;
1572     $res->{index} = $index;
1573
1574     my $error = 0;
1575     foreach my $opt (qw(bps bps_rd bps_wr)) {
1576         if (my $bps = defined(delete $res->{$opt})) {
1577             if (defined($res->{"m$opt"})) {
1578                 warn "both $opt and m$opt specified\n";
1579                 ++$error;
1580                 next;
1581             }
1582             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1583         }
1584     }
1585
1586     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1587     for my $requirement (
1588         [mbps_max => 'mbps'],
1589         [mbps_rd_max => 'mbps_rd'],
1590         [mbps_wr_max => 'mbps_wr'],
1591         [miops_max => 'miops'],
1592         [miops_rd_max => 'miops_rd'],
1593         [miops_wr_max => 'miops_wr'],
1594         [bps_max_length => 'mbps_max'],
1595         [bps_rd_max_length => 'mbps_rd_max'],
1596         [bps_wr_max_length => 'mbps_wr_max'],
1597         [iops_max_length => 'iops_max'],
1598         [iops_rd_max_length => 'iops_rd_max'],
1599         [iops_wr_max_length => 'iops_wr_max']) {
1600         my ($option, $requires) = @$requirement;
1601         if ($res->{$option} && !$res->{$requires}) {
1602             warn "$option requires $requires\n";
1603             ++$error;
1604         }
1605     }
1606
1607     return undef if $error;
1608
1609     return undef if $res->{mbps_rd} && $res->{mbps};
1610     return undef if $res->{mbps_wr} && $res->{mbps};
1611     return undef if $res->{iops_rd} && $res->{iops};
1612     return undef if $res->{iops_wr} && $res->{iops};
1613
1614     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1615         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1616         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1617         return undef if $res->{interface} eq 'virtio';
1618     }
1619
1620     if (my $size = $res->{size}) {
1621         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1622     }
1623
1624     return $res;
1625 }
1626
1627 sub print_drive {
1628     my ($vmid, $drive) = @_;
1629     my $data = { %$drive };
1630     delete $data->{$_} for qw(index interface);
1631     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1632 }
1633
1634 sub scsi_inquiry {
1635     my($fh, $noerr) = @_;
1636
1637     my $SG_IO = 0x2285;
1638     my $SG_GET_VERSION_NUM = 0x2282;
1639
1640     my $versionbuf = "\x00" x 8;
1641     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1642     if (!$ret) {
1643         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1644         return undef;
1645     }
1646     my $version = unpack("I", $versionbuf);
1647     if ($version < 30000) {
1648         die "scsi generic interface too old\n"  if !$noerr;
1649         return undef;
1650     }
1651
1652     my $buf = "\x00" x 36;
1653     my $sensebuf = "\x00" x 8;
1654     my $cmd = pack("C x3 C x1", 0x12, 36);
1655
1656     # see /usr/include/scsi/sg.h
1657     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1658
1659     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1660                       length($sensebuf), 0, length($buf), $buf,
1661                       $cmd, $sensebuf, 6000);
1662
1663     $ret = ioctl($fh, $SG_IO, $packet);
1664     if (!$ret) {
1665         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1666         return undef;
1667     }
1668
1669     my @res = unpack($sg_io_hdr_t, $packet);
1670     if ($res[17] || $res[18]) {
1671         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1672         return undef;
1673     }
1674
1675     my $res = {};
1676     (my $byte0, my $byte1, $res->{vendor},
1677      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1678
1679     $res->{removable} = $byte1 & 128 ? 1 : 0;
1680     $res->{type} = $byte0 & 31;
1681
1682     return $res;
1683 }
1684
1685 sub path_is_scsi {
1686     my ($path) = @_;
1687
1688     my $fh = IO::File->new("+<$path") || return undef;
1689     my $res = scsi_inquiry($fh, 1);
1690     close($fh);
1691
1692     return $res;
1693 }
1694
1695 sub machine_type_is_q35 {
1696     my ($conf) = @_;
1697
1698     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1699 }
1700
1701 sub print_tabletdevice_full {
1702     my ($conf, $arch) = @_;
1703
1704     my $q35 = machine_type_is_q35($conf);
1705
1706     # we use uhci for old VMs because tablet driver was buggy in older qemu
1707     my $usbbus;
1708     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1709         $usbbus = 'ehci';
1710     } else {
1711         $usbbus = 'uhci';
1712     }
1713
1714     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1715 }
1716
1717 sub print_keyboarddevice_full {
1718     my ($conf, $arch, $machine) = @_;
1719
1720     return undef if $arch ne 'aarch64';
1721
1722     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1723 }
1724
1725 sub print_drivedevice_full {
1726     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1727
1728     my $device = '';
1729     my $maxdev = 0;
1730
1731     if ($drive->{interface} eq 'virtio') {
1732         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1733         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1734         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1735     } elsif ($drive->{interface} eq 'scsi') {
1736
1737         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1738         my $unit = $drive->{index} % $maxdev;
1739         my $devicetype = 'hd';
1740         my $path = '';
1741         if (drive_is_cdrom($drive)) {
1742             $devicetype = 'cd';
1743         } else {
1744             if ($drive->{file} =~ m|^/|) {
1745                 $path = $drive->{file};
1746                 if (my $info = path_is_scsi($path)) {
1747                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1748                         $devicetype = 'block';
1749                     } elsif ($info->{type} == 1) { # tape
1750                         $devicetype = 'generic';
1751                     }
1752                 }
1753             } else {
1754                  $path = PVE::Storage::path($storecfg, $drive->{file});
1755             }
1756
1757             if($path =~ m/^iscsi\:\/\//){
1758                 $devicetype = 'generic';
1759             }
1760         }
1761
1762         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1763             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1764         } else {
1765             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1766         }
1767
1768         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1769             $device .= ",rotation_rate=1";
1770         }
1771
1772     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1773         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1774         my $controller = int($drive->{index} / $maxdev);
1775         my $unit = $drive->{index} % $maxdev;
1776         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1777
1778         $device = "ide-$devicetype";
1779         if ($drive->{interface} eq 'ide') {
1780             $device .= ",bus=ide.$controller,unit=$unit";
1781         } else {
1782             $device .= ",bus=ahci$controller.$unit";
1783         }
1784         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1785
1786         if ($devicetype eq 'hd') {
1787             if (my $model = $drive->{model}) {
1788                 $model = URI::Escape::uri_unescape($model);
1789                 $device .= ",model=$model";
1790             }
1791             if ($drive->{ssd}) {
1792                 $device .= ",rotation_rate=1";
1793             }
1794         }
1795     } elsif ($drive->{interface} eq 'usb') {
1796         die "implement me";
1797         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1798     } else {
1799         die "unsupported interface type";
1800     }
1801
1802     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1803
1804     if (my $serial = $drive->{serial}) {
1805         $serial = URI::Escape::uri_unescape($serial);
1806         $device .= ",serial=$serial";
1807     }
1808
1809
1810     return $device;
1811 }
1812
1813 sub get_initiator_name {
1814     my $initiator;
1815
1816     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1817     while (defined(my $line = <$fh>)) {
1818         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1819         $initiator = $1;
1820         last;
1821     }
1822     $fh->close();
1823
1824     return $initiator;
1825 }
1826
1827 sub print_drive_full {
1828     my ($storecfg, $vmid, $drive) = @_;
1829
1830     my $path;
1831     my $volid = $drive->{file};
1832     my $format;
1833
1834     if (drive_is_cdrom($drive)) {
1835         $path = get_iso_path($storecfg, $vmid, $volid);
1836     } else {
1837         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1838         if ($storeid) {
1839             $path = PVE::Storage::path($storecfg, $volid);
1840             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1841             $format = qemu_img_format($scfg, $volname);
1842         } else {
1843             $path = $volid;
1844             $format = "raw";
1845         }
1846    }
1847
1848     my $opts = '';
1849     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1850     foreach my $o (@qemu_drive_options) {
1851         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1852     }
1853
1854     # snapshot only accepts on|off
1855     if (defined($drive->{snapshot})) {
1856         my $v = $drive->{snapshot} ? 'on' : 'off';
1857         $opts .= ",snapshot=$v";
1858     }
1859
1860     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1861         my ($dir, $qmpname) = @$type;
1862         if (my $v = $drive->{"mbps$dir"}) {
1863             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1864         }
1865         if (my $v = $drive->{"mbps${dir}_max"}) {
1866             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1867         }
1868         if (my $v = $drive->{"bps${dir}_max_length"}) {
1869             $opts .= ",throttling.bps$qmpname-max-length=$v";
1870         }
1871         if (my $v = $drive->{"iops${dir}"}) {
1872             $opts .= ",throttling.iops$qmpname=$v";
1873         }
1874         if (my $v = $drive->{"iops${dir}_max"}) {
1875             $opts .= ",throttling.iops$qmpname-max=$v";
1876         }
1877         if (my $v = $drive->{"iops${dir}_max_length"}) {
1878             $opts .= ",throttling.iops$qmpname-max-length=$v";
1879         }
1880     }
1881
1882     $opts .= ",format=$format" if $format && !$drive->{format};
1883
1884     my $cache_direct = 0;
1885
1886     if (my $cache = $drive->{cache}) {
1887         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1888     } elsif (!drive_is_cdrom($drive)) {
1889         $opts .= ",cache=none";
1890         $cache_direct = 1;
1891     }
1892
1893     # aio native works only with O_DIRECT
1894     if (!$drive->{aio}) {
1895         if($cache_direct) {
1896             $opts .= ",aio=native";
1897         } else {
1898             $opts .= ",aio=threads";
1899         }
1900     }
1901
1902     if (!drive_is_cdrom($drive)) {
1903         my $detectzeroes;
1904         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1905             $detectzeroes = 'off';
1906         } elsif ($drive->{discard}) {
1907             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1908         } else {
1909             # This used to be our default with discard not being specified:
1910             $detectzeroes = 'on';
1911         }
1912         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1913     }
1914
1915     my $pathinfo = $path ? "file=$path," : '';
1916
1917     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1918 }
1919
1920 sub print_netdevice_full {
1921     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1922
1923     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1924
1925     my $device = $net->{model};
1926     if ($net->{model} eq 'virtio') {
1927          $device = 'virtio-net-pci';
1928      };
1929
1930     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1931     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1932     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1933         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1934         my $vectors = $net->{queues} * 2 + 2;
1935         $tmpstr .= ",vectors=$vectors,mq=on";
1936     }
1937     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1938
1939     if ($use_old_bios_files) {
1940         my $romfile;
1941         if ($device eq 'virtio-net-pci') {
1942             $romfile = 'pxe-virtio.rom';
1943         } elsif ($device eq 'e1000') {
1944             $romfile = 'pxe-e1000.rom';
1945         } elsif ($device eq 'ne2k') {
1946             $romfile = 'pxe-ne2k_pci.rom';
1947         } elsif ($device eq 'pcnet') {
1948             $romfile = 'pxe-pcnet.rom';
1949         } elsif ($device eq 'rtl8139') {
1950             $romfile = 'pxe-rtl8139.rom';
1951         }
1952         $tmpstr .= ",romfile=$romfile" if $romfile;
1953     }
1954
1955     return $tmpstr;
1956 }
1957
1958 sub print_netdev_full {
1959     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1960
1961     my $i = '';
1962     if ($netid =~ m/^net(\d+)$/) {
1963         $i = int($1);
1964     }
1965
1966     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1967
1968     my $ifname = "tap${vmid}i$i";
1969
1970     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1971     die "interface name '$ifname' is too long (max 15 character)\n"
1972         if length($ifname) >= 16;
1973
1974     my $vhostparam = '';
1975     if (is_native($arch)) {
1976         $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1977     }
1978
1979     my $vmname = $conf->{name} || "vm$vmid";
1980
1981     my $netdev = "";
1982     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1983
1984     if ($net->{bridge}) {
1985         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1986     } else {
1987         $netdev = "type=user,id=$netid,hostname=$vmname";
1988     }
1989
1990     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1991
1992     return $netdev;
1993 }
1994
1995
1996 sub print_cpu_device {
1997     my ($conf, $id) = @_;
1998
1999     my $kvm = $conf->{kvm} // 1;
2000     my $cpu = $kvm ? "kvm64" : "qemu64";
2001     if (my $cputype = $conf->{cpu}) {
2002         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2003             or die "Cannot parse cpu description: $cputype\n";
2004         $cpu = $cpuconf->{cputype};
2005     }
2006
2007     my $cores = $conf->{cores} || 1;
2008
2009     my $current_core = ($id - 1) % $cores;
2010     my $current_socket = int(($id - 1 - $current_core)/$cores);
2011
2012     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2013 }
2014
2015 my $vga_map = {
2016     'cirrus' => 'cirrus-vga',
2017     'std' => 'VGA',
2018     'vmware' => 'vmware-svga',
2019     'virtio' => 'virtio-vga',
2020 };
2021
2022 sub print_vga_device {
2023     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2024
2025     my $type = $vga_map->{$vga->{type}};
2026     if ($type eq 'virtio-vga' && $arch eq 'aarch64') {
2027         $type = 'virtio-gpu';
2028     }
2029     my $vgamem_mb = $vga->{memory};
2030     if ($qxlnum) {
2031         $type = $id ? 'qxl' : 'qxl-vga';
2032     }
2033     die "no devicetype for $vga->{type}\n" if !$type;
2034
2035     my $memory = "";
2036     if ($vgamem_mb) {
2037         if ($vga->{type} eq 'virtio') {
2038             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2039             $memory = ",max_hostmem=$bytes";
2040         } elsif ($qxlnum) {
2041             # from https://www.spice-space.org/multiple-monitors.html
2042             $memory = ",vgamem_mb=$vga->{memory}";
2043             my $ram = $vgamem_mb * 4;
2044             my $vram = $vgamem_mb * 2;
2045             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2046         } else {
2047             $memory = ",vgamem_mb=$vga->{memory}";
2048         }
2049     } elsif ($qxlnum && $id) {
2050         $memory = ",ram_size=67108864,vram_size=33554432";
2051     }
2052
2053     my $q35 = machine_type_is_q35($conf);
2054     my $vgaid = "vga" . ($id // '');
2055     my $pciaddr;
2056
2057     if ($q35 && $vgaid eq 'vga') {
2058         # the first display uses pcie.0 bus on q35 machines
2059         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2060     } else {
2061         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2062     }
2063
2064     return "$type,id=${vgaid}${memory}${pciaddr}";
2065 }
2066
2067 sub drive_is_cloudinit {
2068     my ($drive) = @_;
2069     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2070 }
2071
2072 sub drive_is_cdrom {
2073     my ($drive, $exclude_cloudinit) = @_;
2074
2075     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2076
2077     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2078
2079 }
2080
2081 sub parse_number_sets {
2082     my ($set) = @_;
2083     my $res = [];
2084     foreach my $part (split(/;/, $set)) {
2085         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2086             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2087             push @$res, [ $1, $2 ];
2088         } else {
2089             die "invalid range: $part\n";
2090         }
2091     }
2092     return $res;
2093 }
2094
2095 sub parse_numa {
2096     my ($data) = @_;
2097
2098     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2099     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2100     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2101     return $res;
2102 }
2103
2104 sub parse_hostpci {
2105     my ($value) = @_;
2106
2107     return undef if !$value;
2108
2109     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2110
2111     my @idlist = split(/;/, $res->{host});
2112     delete $res->{host};
2113     foreach my $id (@idlist) {
2114         if ($id =~ m/\./) { # full id 00:00.1
2115             push @{$res->{pciid}}, {
2116                 id => $id,
2117             };
2118         } else { # partial id 00:00
2119             $res->{pciid} = PVE::SysFSTools::lspci($id);
2120         }
2121     }
2122     return $res;
2123 }
2124
2125 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2126 sub parse_net {
2127     my ($data) = @_;
2128
2129     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2130     if ($@) {
2131         warn $@;
2132         return undef;
2133     }
2134     if (!defined($res->{macaddr})) {
2135         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2136         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2137     }
2138     return $res;
2139 }
2140
2141 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2142 sub parse_ipconfig {
2143     my ($data) = @_;
2144
2145     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2146     if ($@) {
2147         warn $@;
2148         return undef;
2149     }
2150
2151     if ($res->{gw} && !$res->{ip}) {
2152         warn 'gateway specified without specifying an IP address';
2153         return undef;
2154     }
2155     if ($res->{gw6} && !$res->{ip6}) {
2156         warn 'IPv6 gateway specified without specifying an IPv6 address';
2157         return undef;
2158     }
2159     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2160         warn 'gateway specified together with DHCP';
2161         return undef;
2162     }
2163     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2164         # gw6 + auto/dhcp
2165         warn "IPv6 gateway specified together with $res->{ip6} address";
2166         return undef;
2167     }
2168
2169     if (!$res->{ip} && !$res->{ip6}) {
2170         return { ip => 'dhcp', ip6 => 'dhcp' };
2171     }
2172
2173     return $res;
2174 }
2175
2176 sub print_net {
2177     my $net = shift;
2178
2179     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2180 }
2181
2182 sub add_random_macs {
2183     my ($settings) = @_;
2184
2185     foreach my $opt (keys %$settings) {
2186         next if $opt !~ m/^net(\d+)$/;
2187         my $net = parse_net($settings->{$opt});
2188         next if !$net;
2189         $settings->{$opt} = print_net($net);
2190     }
2191 }
2192
2193 sub vm_is_volid_owner {
2194     my ($storecfg, $vmid, $volid) = @_;
2195
2196     if ($volid !~  m|^/|) {
2197         my ($path, $owner);
2198         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2199         if ($owner && ($owner == $vmid)) {
2200             return 1;
2201         }
2202     }
2203
2204     return undef;
2205 }
2206
2207 sub split_flagged_list {
2208     my $text = shift || '';
2209     $text =~ s/[,;]/ /g;
2210     $text =~ s/^\s+//;
2211     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2212 }
2213
2214 sub join_flagged_list {
2215     my ($how, $lst) = @_;
2216     join $how, map { $lst->{$_} . $_ } keys %$lst;
2217 }
2218
2219 sub vmconfig_delete_pending_option {
2220     my ($conf, $key, $force) = @_;
2221
2222     delete $conf->{pending}->{$key};
2223     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2224     $pending_delete_hash->{$key} = $force ? '!' : '';
2225     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2226 }
2227
2228 sub vmconfig_undelete_pending_option {
2229     my ($conf, $key) = @_;
2230
2231     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2232     delete $pending_delete_hash->{$key};
2233
2234     if (%$pending_delete_hash) {
2235         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2236     } else {
2237         delete $conf->{pending}->{delete};
2238     }
2239 }
2240
2241 sub vmconfig_register_unused_drive {
2242     my ($storecfg, $vmid, $conf, $drive) = @_;
2243
2244     if (drive_is_cloudinit($drive)) {
2245         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2246         warn $@ if $@;
2247     } elsif (!drive_is_cdrom($drive)) {
2248         my $volid = $drive->{file};
2249         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2250             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2251         }
2252     }
2253 }
2254
2255 sub vmconfig_cleanup_pending {
2256     my ($conf) = @_;
2257
2258     # remove pending changes when nothing changed
2259     my $changes;
2260     foreach my $opt (keys %{$conf->{pending}}) {
2261         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2262             $changes = 1;
2263             delete $conf->{pending}->{$opt};
2264         }
2265     }
2266
2267     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2268     my $pending_delete_hash = {};
2269     while (my ($opt, $force) = each %$current_delete_hash) {
2270         if (defined($conf->{$opt})) {
2271             $pending_delete_hash->{$opt} = $force;
2272         } else {
2273             $changes = 1;
2274         }
2275     }
2276
2277     if (%$pending_delete_hash) {
2278         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2279     } else {
2280         delete $conf->{pending}->{delete};
2281     }
2282
2283     return $changes;
2284 }
2285
2286 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2287 my $smbios1_fmt = {
2288     uuid => {
2289         type => 'string',
2290         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2291         format_description => 'UUID',
2292         description => "Set SMBIOS1 UUID.",
2293         optional => 1,
2294     },
2295     version => {
2296         type => 'string',
2297         pattern => '\S+',
2298         format_description => 'string',
2299         description => "Set SMBIOS1 version.",
2300         optional => 1,
2301     },
2302     serial => {
2303         type => 'string',
2304         pattern => '\S+',
2305         format_description => 'string',
2306         description => "Set SMBIOS1 serial number.",
2307         optional => 1,
2308     },
2309     manufacturer => {
2310         type => 'string',
2311         pattern => '\S+',
2312         format_description => 'string',
2313         description => "Set SMBIOS1 manufacturer.",
2314         optional => 1,
2315     },
2316     product => {
2317         type => 'string',
2318         pattern => '\S+',
2319         format_description => 'string',
2320         description => "Set SMBIOS1 product ID.",
2321         optional => 1,
2322     },
2323     sku => {
2324         type => 'string',
2325         pattern => '\S+',
2326         format_description => 'string',
2327         description => "Set SMBIOS1 SKU string.",
2328         optional => 1,
2329     },
2330     family => {
2331         type => 'string',
2332         pattern => '\S+',
2333         format_description => 'string',
2334         description => "Set SMBIOS1 family string.",
2335         optional => 1,
2336     },
2337 };
2338
2339 sub parse_smbios1 {
2340     my ($data) = @_;
2341
2342     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2343     warn $@ if $@;
2344     return $res;
2345 }
2346
2347 sub print_smbios1 {
2348     my ($smbios1) = @_;
2349     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2350 }
2351
2352 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2353
2354 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2355 sub verify_bootdisk {
2356     my ($value, $noerr) = @_;
2357
2358     return $value if is_valid_drivename($value);
2359
2360     return undef if $noerr;
2361
2362     die "invalid boot disk '$value'\n";
2363 }
2364
2365 sub parse_watchdog {
2366     my ($value) = @_;
2367
2368     return undef if !$value;
2369
2370     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2371     warn $@ if $@;
2372     return $res;
2373 }
2374
2375 sub parse_guest_agent {
2376     my ($value) = @_;
2377
2378     return {} if !defined($value->{agent});
2379
2380     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2381     warn $@ if $@;
2382
2383     # if the agent is disabled ignore the other potentially set properties
2384     return {} if !$res->{enabled};
2385     return $res;
2386 }
2387
2388 sub parse_vga {
2389     my ($value) = @_;
2390
2391     return {} if !$value;
2392     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2393     warn $@ if $@;
2394     return $res;
2395 }
2396
2397 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2398 sub verify_usb_device {
2399     my ($value, $noerr) = @_;
2400
2401     return $value if parse_usb_device($value);
2402
2403     return undef if $noerr;
2404
2405     die "unable to parse usb device\n";
2406 }
2407
2408 # add JSON properties for create and set function
2409 sub json_config_properties {
2410     my $prop = shift;
2411
2412     foreach my $opt (keys %$confdesc) {
2413         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2414         $prop->{$opt} = $confdesc->{$opt};
2415     }
2416
2417     return $prop;
2418 }
2419
2420 # return copy of $confdesc_cloudinit to generate documentation
2421 sub cloudinit_config_properties {
2422
2423     return dclone($confdesc_cloudinit);
2424 }
2425
2426 sub check_type {
2427     my ($key, $value) = @_;
2428
2429     die "unknown setting '$key'\n" if !$confdesc->{$key};
2430
2431     my $type = $confdesc->{$key}->{type};
2432
2433     if (!defined($value)) {
2434         die "got undefined value\n";
2435     }
2436
2437     if ($value =~ m/[\n\r]/) {
2438         die "property contains a line feed\n";
2439     }
2440
2441     if ($type eq 'boolean') {
2442         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2443         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2444         die "type check ('boolean') failed - got '$value'\n";
2445     } elsif ($type eq 'integer') {
2446         return int($1) if $value =~ m/^(\d+)$/;
2447         die "type check ('integer') failed - got '$value'\n";
2448     } elsif ($type eq 'number') {
2449         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2450         die "type check ('number') failed - got '$value'\n";
2451     } elsif ($type eq 'string') {
2452         if (my $fmt = $confdesc->{$key}->{format}) {
2453             PVE::JSONSchema::check_format($fmt, $value);
2454             return $value;
2455         }
2456         $value =~ s/^\"(.*)\"$/$1/;
2457         return $value;
2458     } else {
2459         die "internal error"
2460     }
2461 }
2462
2463 sub touch_config {
2464     my ($vmid) = @_;
2465
2466     my $conf = PVE::QemuConfig->config_file($vmid);
2467     utime undef, undef, $conf;
2468 }
2469
2470 sub destroy_vm {
2471     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2472
2473     my $conffile = PVE::QemuConfig->config_file($vmid);
2474
2475     my $conf = PVE::QemuConfig->load_config($vmid);
2476
2477     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2478
2479     if ($conf->{template}) {
2480         # check if any base image is still used by a linked clone
2481         foreach_drive($conf, sub {
2482                 my ($ds, $drive) = @_;
2483
2484                 return if drive_is_cdrom($drive);
2485
2486                 my $volid = $drive->{file};
2487
2488                 return if !$volid || $volid =~ m|^/|;
2489
2490                 die "base volume '$volid' is still in use by linked cloned\n"
2491                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2492
2493         });
2494     }
2495
2496     # only remove disks owned by this VM
2497     foreach_drive($conf, sub {
2498         my ($ds, $drive) = @_;
2499
2500         return if drive_is_cdrom($drive, 1);
2501
2502         my $volid = $drive->{file};
2503
2504         return if !$volid || $volid =~ m|^/|;
2505
2506         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2507         return if !$path || !$owner || ($owner != $vmid);
2508
2509         eval {
2510             PVE::Storage::vdisk_free($storecfg, $volid);
2511         };
2512         warn "Could not remove disk '$volid', check manually: $@" if $@;
2513
2514     });
2515
2516     if ($keep_empty_config) {
2517         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2518     } else {
2519         unlink $conffile;
2520     }
2521
2522     # also remove unused disk
2523     eval {
2524         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2525
2526         eval {
2527             PVE::Storage::foreach_volid($dl, sub {
2528                 my ($volid, $sid, $volname, $d) = @_;
2529                 PVE::Storage::vdisk_free($storecfg, $volid);
2530             });
2531         };
2532         warn $@ if $@;
2533
2534     };
2535     warn $@ if $@;
2536 }
2537
2538 sub parse_vm_config {
2539     my ($filename, $raw) = @_;
2540
2541     return undef if !defined($raw);
2542
2543     my $res = {
2544         digest => Digest::SHA::sha1_hex($raw),
2545         snapshots => {},
2546         pending => {},
2547     };
2548
2549     $filename =~ m|/qemu-server/(\d+)\.conf$|
2550         || die "got strange filename '$filename'";
2551
2552     my $vmid = $1;
2553
2554     my $conf = $res;
2555     my $descr;
2556     my $section = '';
2557
2558     my @lines = split(/\n/, $raw);
2559     foreach my $line (@lines) {
2560         next if $line =~ m/^\s*$/;
2561
2562         if ($line =~ m/^\[PENDING\]\s*$/i) {
2563             $section = 'pending';
2564             if (defined($descr)) {
2565                 $descr =~ s/\s+$//;
2566                 $conf->{description} = $descr;
2567             }
2568             $descr = undef;
2569             $conf = $res->{$section} = {};
2570             next;
2571
2572         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2573             $section = $1;
2574             if (defined($descr)) {
2575                 $descr =~ s/\s+$//;
2576                 $conf->{description} = $descr;
2577             }
2578             $descr = undef;
2579             $conf = $res->{snapshots}->{$section} = {};
2580             next;
2581         }
2582
2583         if ($line =~ m/^\#(.*)\s*$/) {
2584             $descr = '' if !defined($descr);
2585             $descr .= PVE::Tools::decode_text($1) . "\n";
2586             next;
2587         }
2588
2589         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2590             $descr = '' if !defined($descr);
2591             $descr .= PVE::Tools::decode_text($2);
2592         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2593             $conf->{snapstate} = $1;
2594         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2595             my $key = $1;
2596             my $value = $2;
2597             $conf->{$key} = $value;
2598         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2599             my $value = $1;
2600             if ($section eq 'pending') {
2601                 $conf->{delete} = $value; # we parse this later
2602             } else {
2603                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2604             }
2605         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2606             my $key = $1;
2607             my $value = $2;
2608             eval { $value = check_type($key, $value); };
2609             if ($@) {
2610                 warn "vm $vmid - unable to parse value of '$key' - $@";
2611             } else {
2612                 $key = 'ide2' if $key eq 'cdrom';
2613                 my $fmt = $confdesc->{$key}->{format};
2614                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2615                     my $v = parse_drive($key, $value);
2616                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2617                         $v->{file} = $volid;
2618                         $value = print_drive($vmid, $v);
2619                     } else {
2620                         warn "vm $vmid - unable to parse value of '$key'\n";
2621                         next;
2622                     }
2623                 }
2624
2625                 $conf->{$key} = $value;
2626             }
2627         }
2628     }
2629
2630     if (defined($descr)) {
2631         $descr =~ s/\s+$//;
2632         $conf->{description} = $descr;
2633     }
2634     delete $res->{snapstate}; # just to be sure
2635
2636     return $res;
2637 }
2638
2639 sub write_vm_config {
2640     my ($filename, $conf) = @_;
2641
2642     delete $conf->{snapstate}; # just to be sure
2643
2644     if ($conf->{cdrom}) {
2645         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2646         $conf->{ide2} = $conf->{cdrom};
2647         delete $conf->{cdrom};
2648     }
2649
2650     # we do not use 'smp' any longer
2651     if ($conf->{sockets}) {
2652         delete $conf->{smp};
2653     } elsif ($conf->{smp}) {
2654         $conf->{sockets} = $conf->{smp};
2655         delete $conf->{cores};
2656         delete $conf->{smp};
2657     }
2658
2659     my $used_volids = {};
2660
2661     my $cleanup_config = sub {
2662         my ($cref, $pending, $snapname) = @_;
2663
2664         foreach my $key (keys %$cref) {
2665             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2666                 $key eq 'snapstate' || $key eq 'pending';
2667             my $value = $cref->{$key};
2668             if ($key eq 'delete') {
2669                 die "propertry 'delete' is only allowed in [PENDING]\n"
2670                     if !$pending;
2671                 # fixme: check syntax?
2672                 next;
2673             }
2674             eval { $value = check_type($key, $value); };
2675             die "unable to parse value of '$key' - $@" if $@;
2676
2677             $cref->{$key} = $value;
2678
2679             if (!$snapname && is_valid_drivename($key)) {
2680                 my $drive = parse_drive($key, $value);
2681                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2682             }
2683         }
2684     };
2685
2686     &$cleanup_config($conf);
2687
2688     &$cleanup_config($conf->{pending}, 1);
2689
2690     foreach my $snapname (keys %{$conf->{snapshots}}) {
2691         die "internal error" if $snapname eq 'pending';
2692         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2693     }
2694
2695     # remove 'unusedX' settings if we re-add a volume
2696     foreach my $key (keys %$conf) {
2697         my $value = $conf->{$key};
2698         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2699             delete $conf->{$key};
2700         }
2701     }
2702
2703     my $generate_raw_config = sub {
2704         my ($conf, $pending) = @_;
2705
2706         my $raw = '';
2707
2708         # add description as comment to top of file
2709         if (defined(my $descr = $conf->{description})) {
2710             if ($descr) {
2711                 foreach my $cl (split(/\n/, $descr)) {
2712                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2713                 }
2714             } else {
2715                 $raw .= "#\n" if $pending;
2716             }
2717         }
2718
2719         foreach my $key (sort keys %$conf) {
2720             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2721             $raw .= "$key: $conf->{$key}\n";
2722         }
2723         return $raw;
2724     };
2725
2726     my $raw = &$generate_raw_config($conf);
2727
2728     if (scalar(keys %{$conf->{pending}})){
2729         $raw .= "\n[PENDING]\n";
2730         $raw .= &$generate_raw_config($conf->{pending}, 1);
2731     }
2732
2733     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2734         $raw .= "\n[$snapname]\n";
2735         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2736     }
2737
2738     return $raw;
2739 }
2740
2741 sub load_defaults {
2742
2743     my $res = {};
2744
2745     # we use static defaults from our JSON schema configuration
2746     foreach my $key (keys %$confdesc) {
2747         if (defined(my $default = $confdesc->{$key}->{default})) {
2748             $res->{$key} = $default;
2749         }
2750     }
2751
2752     return $res;
2753 }
2754
2755 sub config_list {
2756     my $vmlist = PVE::Cluster::get_vmlist();
2757     my $res = {};
2758     return $res if !$vmlist || !$vmlist->{ids};
2759     my $ids = $vmlist->{ids};
2760
2761     foreach my $vmid (keys %$ids) {
2762         my $d = $ids->{$vmid};
2763         next if !$d->{node} || $d->{node} ne $nodename;
2764         next if !$d->{type} || $d->{type} ne 'qemu';
2765         $res->{$vmid}->{exists} = 1;
2766     }
2767     return $res;
2768 }
2769
2770 # test if VM uses local resources (to prevent migration)
2771 sub check_local_resources {
2772     my ($conf, $noerr) = @_;
2773
2774     my $loc_res = 0;
2775
2776     $loc_res = 1 if $conf->{hostusb}; # old syntax
2777     $loc_res = 1 if $conf->{hostpci}; # old syntax
2778
2779     foreach my $k (keys %$conf) {
2780         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2781         # sockets are safe: they will recreated be on the target side post-migrate
2782         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2783         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2784     }
2785
2786     die "VM uses local resources\n" if $loc_res && !$noerr;
2787
2788     return $loc_res;
2789 }
2790
2791 # check if used storages are available on all nodes (use by migrate)
2792 sub check_storage_availability {
2793     my ($storecfg, $conf, $node) = @_;
2794
2795     foreach_drive($conf, sub {
2796         my ($ds, $drive) = @_;
2797
2798         my $volid = $drive->{file};
2799         return if !$volid;
2800
2801         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2802         return if !$sid;
2803
2804         # check if storage is available on both nodes
2805         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2806         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2807    });
2808 }
2809
2810 # list nodes where all VM images are available (used by has_feature API)
2811 sub shared_nodes {
2812     my ($conf, $storecfg) = @_;
2813
2814     my $nodelist = PVE::Cluster::get_nodelist();
2815     my $nodehash = { map { $_ => 1 } @$nodelist };
2816     my $nodename = PVE::INotify::nodename();
2817
2818     foreach_drive($conf, sub {
2819         my ($ds, $drive) = @_;
2820
2821         my $volid = $drive->{file};
2822         return if !$volid;
2823
2824         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2825         if ($storeid) {
2826             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2827             if ($scfg->{disable}) {
2828                 $nodehash = {};
2829             } elsif (my $avail = $scfg->{nodes}) {
2830                 foreach my $node (keys %$nodehash) {
2831                     delete $nodehash->{$node} if !$avail->{$node};
2832                 }
2833             } elsif (!$scfg->{shared}) {
2834                 foreach my $node (keys %$nodehash) {
2835                     delete $nodehash->{$node} if $node ne $nodename
2836                 }
2837             }
2838         }
2839     });
2840
2841     return $nodehash
2842 }
2843
2844 sub check_cmdline {
2845     my ($pidfile, $pid) = @_;
2846
2847     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2848     if (defined($fh)) {
2849         my $line = <$fh>;
2850         $fh->close;
2851         return undef if !$line;
2852         my @param = split(/\0/, $line);
2853
2854         my $cmd = $param[0];
2855         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2856
2857         for (my $i = 0; $i < scalar (@param); $i++) {
2858             my $p = $param[$i];
2859             next if !$p;
2860             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2861                 my $p = $param[$i+1];
2862                 return 1 if $p && ($p eq $pidfile);
2863                 return undef;
2864             }
2865         }
2866     }
2867     return undef;
2868 }
2869
2870 sub check_running {
2871     my ($vmid, $nocheck, $node) = @_;
2872
2873     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2874
2875     die "unable to find configuration file for VM $vmid - no such machine\n"
2876         if !$nocheck && ! -f $filename;
2877
2878     my $pidfile = pidfile_name($vmid);
2879
2880     if (my $fd = IO::File->new("<$pidfile")) {
2881         my $st = stat($fd);
2882         my $line = <$fd>;
2883         close($fd);
2884
2885         my $mtime = $st->mtime;
2886         if ($mtime > time()) {
2887             warn "file '$filename' modified in future\n";
2888         }
2889
2890         if ($line =~ m/^(\d+)$/) {
2891             my $pid = $1;
2892             if (check_cmdline($pidfile, $pid)) {
2893                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2894                     return $pid;
2895                 }
2896             }
2897         }
2898     }
2899
2900     return undef;
2901 }
2902
2903 sub vzlist {
2904
2905     my $vzlist = config_list();
2906
2907     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2908
2909     while (defined(my $de = $fd->read)) {
2910         next if $de !~ m/^(\d+)\.pid$/;
2911         my $vmid = $1;
2912         next if !defined($vzlist->{$vmid});
2913         if (my $pid = check_running($vmid)) {
2914             $vzlist->{$vmid}->{pid} = $pid;
2915         }
2916     }
2917
2918     return $vzlist;
2919 }
2920
2921 sub disksize {
2922     my ($storecfg, $conf) = @_;
2923
2924     my $bootdisk = $conf->{bootdisk};
2925     return undef if !$bootdisk;
2926     return undef if !is_valid_drivename($bootdisk);
2927
2928     return undef if !$conf->{$bootdisk};
2929
2930     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2931     return undef if !defined($drive);
2932
2933     return undef if drive_is_cdrom($drive);
2934
2935     my $volid = $drive->{file};
2936     return undef if !$volid;
2937
2938     return $drive->{size};
2939 }
2940
2941 our $vmstatus_return_properties = {
2942     vmid => get_standard_option('pve-vmid'),
2943     status => {
2944         description => "Qemu process status.",
2945         type => 'string',
2946         enum => ['stopped', 'running'],
2947     },
2948     maxmem => {
2949         description => "Maximum memory in bytes.",
2950         type => 'integer',
2951         optional => 1,
2952         renderer => 'bytes',
2953     },
2954     maxdisk => {
2955         description => "Root disk size in bytes.",
2956         type => 'integer',
2957         optional => 1,
2958         renderer => 'bytes',
2959     },
2960     name => {
2961         description => "VM name.",
2962         type => 'string',
2963         optional => 1,
2964     },
2965     qmpstatus => {
2966         description => "Qemu QMP agent status.",
2967         type => 'string',
2968         optional => 1,
2969     },
2970     pid => {
2971         description => "PID of running qemu process.",
2972         type => 'integer',
2973         optional => 1,
2974     },
2975     uptime => {
2976         description => "Uptime.",
2977         type => 'integer',
2978         optional => 1,
2979         renderer => 'duration',
2980     },
2981     cpus => {
2982         description => "Maximum usable CPUs.",
2983         type => 'number',
2984         optional => 1,
2985     },
2986 };
2987
2988 my $last_proc_pid_stat;
2989
2990 # get VM status information
2991 # This must be fast and should not block ($full == false)
2992 # We only query KVM using QMP if $full == true (this can be slow)
2993 sub vmstatus {
2994     my ($opt_vmid, $full) = @_;
2995
2996     my $res = {};
2997
2998     my $storecfg = PVE::Storage::config();
2999
3000     my $list = vzlist();
3001     my $defaults = load_defaults();
3002
3003     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3004
3005     my $cpucount = $cpuinfo->{cpus} || 1;
3006
3007     foreach my $vmid (keys %$list) {
3008         next if $opt_vmid && ($vmid ne $opt_vmid);
3009
3010         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3011         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3012
3013         my $d = { vmid => $vmid };
3014         $d->{pid} = $list->{$vmid}->{pid};
3015
3016         # fixme: better status?
3017         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3018
3019         my $size = disksize($storecfg, $conf);
3020         if (defined($size)) {
3021             $d->{disk} = 0; # no info available
3022             $d->{maxdisk} = $size;
3023         } else {
3024             $d->{disk} = 0;
3025             $d->{maxdisk} = 0;
3026         }
3027
3028         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3029             * ($conf->{cores} || $defaults->{cores});
3030         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3031         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3032
3033         $d->{name} = $conf->{name} || "VM $vmid";
3034         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3035             : $defaults->{memory}*(1024*1024);
3036
3037         if ($conf->{balloon}) {
3038             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3039             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3040                 : $defaults->{shares};
3041         }
3042
3043         $d->{uptime} = 0;
3044         $d->{cpu} = 0;
3045         $d->{mem} = 0;
3046
3047         $d->{netout} = 0;
3048         $d->{netin} = 0;
3049
3050         $d->{diskread} = 0;
3051         $d->{diskwrite} = 0;
3052
3053         $d->{template} = PVE::QemuConfig->is_template($conf);
3054
3055         $d->{serial} = 1 if conf_has_serial($conf);
3056
3057         $res->{$vmid} = $d;
3058     }
3059
3060     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3061     foreach my $dev (keys %$netdev) {
3062         next if $dev !~ m/^tap([1-9]\d*)i/;
3063         my $vmid = $1;
3064         my $d = $res->{$vmid};
3065         next if !$d;
3066
3067         $d->{netout} += $netdev->{$dev}->{receive};
3068         $d->{netin} += $netdev->{$dev}->{transmit};
3069
3070         if ($full) {
3071             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3072             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3073         }
3074
3075     }
3076
3077     my $ctime = gettimeofday;
3078
3079     foreach my $vmid (keys %$list) {
3080
3081         my $d = $res->{$vmid};
3082         my $pid = $d->{pid};
3083         next if !$pid;
3084
3085         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3086         next if !$pstat; # not running
3087
3088         my $used = $pstat->{utime} + $pstat->{stime};
3089
3090         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3091
3092         if ($pstat->{vsize}) {
3093             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3094         }
3095
3096         my $old = $last_proc_pid_stat->{$pid};
3097         if (!$old) {
3098             $last_proc_pid_stat->{$pid} = {
3099                 time => $ctime,
3100                 used => $used,
3101                 cpu => 0,
3102             };
3103             next;
3104         }
3105
3106         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3107
3108         if ($dtime > 1000) {
3109             my $dutime = $used -  $old->{used};
3110
3111             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3112             $last_proc_pid_stat->{$pid} = {
3113                 time => $ctime,
3114                 used => $used,
3115                 cpu => $d->{cpu},
3116             };
3117         } else {
3118             $d->{cpu} = $old->{cpu};
3119         }
3120     }
3121
3122     return $res if !$full;
3123
3124     my $qmpclient = PVE::QMPClient->new();
3125
3126     my $ballooncb = sub {
3127         my ($vmid, $resp) = @_;
3128
3129         my $info = $resp->{'return'};
3130         return if !$info->{max_mem};
3131
3132         my $d = $res->{$vmid};
3133
3134         # use memory assigned to VM
3135         $d->{maxmem} = $info->{max_mem};
3136         $d->{balloon} = $info->{actual};
3137
3138         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3139             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3140             $d->{freemem} = $info->{free_mem};
3141         }
3142
3143         $d->{ballooninfo} = $info;
3144     };
3145
3146     my $blockstatscb = sub {
3147         my ($vmid, $resp) = @_;
3148         my $data = $resp->{'return'} || [];
3149         my $totalrdbytes = 0;
3150         my $totalwrbytes = 0;
3151
3152         for my $blockstat (@$data) {
3153             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3154             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3155
3156             $blockstat->{device} =~ s/drive-//;
3157             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3158         }
3159         $res->{$vmid}->{diskread} = $totalrdbytes;
3160         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3161     };
3162
3163     my $statuscb = sub {
3164         my ($vmid, $resp) = @_;
3165
3166         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3167         # this fails if ballon driver is not loaded, so this must be
3168         # the last commnand (following command are aborted if this fails).
3169         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3170
3171         my $status = 'unknown';
3172         if (!defined($status = $resp->{'return'}->{status})) {
3173             warn "unable to get VM status\n";
3174             return;
3175         }
3176
3177         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3178     };
3179
3180     foreach my $vmid (keys %$list) {
3181         next if $opt_vmid && ($vmid ne $opt_vmid);
3182         next if !$res->{$vmid}->{pid}; # not running
3183         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3184     }
3185
3186     $qmpclient->queue_execute(undef, 2);
3187
3188     foreach my $vmid (keys %$list) {
3189         next if $opt_vmid && ($vmid ne $opt_vmid);
3190         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3191     }
3192
3193     return $res;
3194 }
3195
3196 sub foreach_drive {
3197     my ($conf, $func, @param) = @_;
3198
3199     foreach my $ds (valid_drive_names()) {
3200         next if !defined($conf->{$ds});
3201
3202         my $drive = parse_drive($ds, $conf->{$ds});
3203         next if !$drive;
3204
3205         &$func($ds, $drive, @param);
3206     }
3207 }
3208
3209 sub foreach_volid {
3210     my ($conf, $func, @param) = @_;
3211
3212     my $volhash = {};
3213
3214     my $test_volid = sub {
3215         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3216
3217         return if !$volid;
3218
3219         $volhash->{$volid}->{cdrom} //= 1;
3220         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3221
3222         $volhash->{$volid}->{replicate} //= 0;
3223         $volhash->{$volid}->{replicate} = 1 if $replicate;
3224
3225         $volhash->{$volid}->{shared} //= 0;
3226         $volhash->{$volid}->{shared} = 1 if $shared;
3227
3228         $volhash->{$volid}->{referenced_in_config} //= 0;
3229         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3230
3231         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3232             if defined($snapname);
3233     };
3234
3235     foreach_drive($conf, sub {
3236         my ($ds, $drive) = @_;
3237         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3238     });
3239
3240     foreach my $snapname (keys %{$conf->{snapshots}}) {
3241         my $snap = $conf->{snapshots}->{$snapname};
3242         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3243         foreach_drive($snap, sub {
3244             my ($ds, $drive) = @_;
3245             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3246         });
3247     }
3248
3249     foreach my $volid (keys %$volhash) {
3250         &$func($volid, $volhash->{$volid}, @param);
3251     }
3252 }
3253
3254 sub conf_has_serial {
3255     my ($conf) = @_;
3256
3257     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3258         if ($conf->{"serial$i"}) {
3259             return 1;
3260         }
3261     }
3262
3263     return 0;
3264 }
3265
3266 sub vga_conf_has_spice {
3267     my ($vga) = @_;
3268
3269     my $vgaconf = parse_vga($vga);
3270     my $vgatype = $vgaconf->{type};
3271     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3272
3273     return $1 || 1;
3274 }
3275
3276 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3277 sub get_host_arch() {
3278     $host_arch = (POSIX::uname())[4] if !$host_arch;
3279     return $host_arch;
3280 }
3281
3282 sub is_native($) {
3283     my ($arch) = @_;
3284     return get_host_arch() eq $arch;
3285 }
3286
3287 my $default_machines = {
3288     x86_64 => 'pc',
3289     aarch64 => 'virt',
3290 };
3291
3292 sub get_basic_machine_info {
3293     my ($conf, $forcemachine) = @_;
3294
3295     my $arch = $conf->{arch} // get_host_arch();
3296     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3297     return ($arch, $machine);
3298 }
3299
3300 sub get_ovmf_files($) {
3301     my ($arch) = @_;
3302
3303     my $ovmf = $OVMF->{$arch}
3304         or die "no OVMF images known for architecture '$arch'\n";
3305
3306     return @$ovmf;
3307 }
3308
3309 my $Arch2Qemu = {
3310     aarch64 => '/usr/bin/qemu-system-aarch64',
3311     x86_64 => '/usr/bin/qemu-system-x86_64',
3312 };
3313 sub get_command_for_arch($) {
3314     my ($arch) = @_;
3315     return '/usr/bin/kvm' if is_native($arch);
3316
3317     my $cmd = $Arch2Qemu->{$arch}
3318         or die "don't know how to emulate architecture '$arch'\n";
3319     return $cmd;
3320 }
3321
3322 sub get_cpu_options {
3323     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3324
3325     my $cpuFlags = [];
3326     my $ostype = $conf->{ostype};
3327
3328     my $cpu = $kvm ? "kvm64" : "qemu64";
3329     if ($arch eq 'aarch64') {
3330         $cpu = 'cortex-a57';
3331     }
3332     if (my $cputype = $conf->{cpu}) {
3333         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3334             or die "Cannot parse cpu description: $cputype\n";
3335         $cpu = $cpuconf->{cputype};
3336         $kvm_off = 1 if $cpuconf->{hidden};
3337
3338         if (defined(my $flags = $cpuconf->{flags})) {
3339             push @$cpuFlags, split(";", $flags);
3340         }
3341     }
3342
3343     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3344
3345     push @$cpuFlags , '-x2apic'
3346         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3347
3348     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3349
3350     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3351
3352     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3353
3354         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3355         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3356     }
3357
3358     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3359
3360     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3361
3362     push @$cpuFlags, 'kvm=off' if $kvm_off;
3363
3364     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3365         push @$cpuFlags, "vendor=${cpu_vendor}"
3366             if $cpu_vendor ne 'default';
3367     } elsif ($arch ne 'aarch64') {
3368         die "internal error"; # should not happen
3369     }
3370
3371     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3372
3373     return ('-cpu', $cpu);
3374 }
3375
3376 sub config_to_command {
3377     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3378
3379     my $cmd = [];
3380     my $globalFlags = [];
3381     my $machineFlags = [];
3382     my $rtcFlags = [];
3383     my $devices = [];
3384     my $pciaddr = '';
3385     my $bridges = {};
3386     my $kvmver = kvm_user_version();
3387     my $vernum = 0; # unknown
3388     my $ostype = $conf->{ostype};
3389     my $winversion = windows_version($ostype);
3390     my $kvm = $conf->{kvm};
3391
3392     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3393     $kvm //= 1 if is_native($arch);
3394
3395     if ($kvm) {
3396         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3397             if !defined kvm_version();
3398     }
3399
3400     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3401         $vernum = $1*1000000+$2*1000;
3402     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3403         $vernum = $1*1000000+$2*1000+$3;
3404     }
3405
3406     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3407
3408     my $have_ovz = -f '/proc/vz/vestat';
3409
3410     my $q35 = machine_type_is_q35($conf);
3411     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3412     my $use_old_bios_files = undef;
3413     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3414
3415     my $cpuunits = defined($conf->{cpuunits}) ?
3416             $conf->{cpuunits} : $defaults->{cpuunits};
3417
3418     push @$cmd, get_command_for_arch($arch);
3419
3420     push @$cmd, '-id', $vmid;
3421
3422     my $vmname = $conf->{name} || "vm$vmid";
3423
3424     push @$cmd, '-name', $vmname;
3425
3426     my $use_virtio = 0;
3427
3428     my $qmpsocket = qmp_socket($vmid);
3429     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3430     push @$cmd, '-mon', "chardev=qmp,mode=control";
3431
3432     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3433         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3434         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3435     }
3436
3437     push @$cmd, '-pidfile' , pidfile_name($vmid);
3438
3439     push @$cmd, '-daemonize';
3440
3441     if ($conf->{smbios1}) {
3442         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3443     }
3444
3445     if ($conf->{vmgenid}) {
3446         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3447     }
3448
3449     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3450     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3451         die "uefi base image not found\n" if ! -f $ovmf_code;
3452
3453         my $path;
3454         my $format;
3455         if (my $efidisk = $conf->{efidisk0}) {
3456             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3457             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3458             $format = $d->{format};
3459             if ($storeid) {
3460                 $path = PVE::Storage::path($storecfg, $d->{file});
3461                 if (!defined($format)) {
3462                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3463                     $format = qemu_img_format($scfg, $volname);
3464                 }
3465             } else {
3466                 $path = $d->{file};
3467                 die "efidisk format must be specified\n"
3468                     if !defined($format);
3469             }
3470         } else {
3471             warn "no efidisk configured! Using temporary efivars disk.\n";
3472             $path = "/tmp/$vmid-ovmf.fd";
3473             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3474             $format = 'raw';
3475         }
3476
3477         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3478         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3479     }
3480
3481
3482     # add usb controllers
3483     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3484     push @$devices, @usbcontrollers if @usbcontrollers;
3485     my $vga = parse_vga($conf->{vga});
3486
3487     my $qxlnum = vga_conf_has_spice($conf->{vga});
3488     $vga->{type} = 'qxl' if $qxlnum;
3489
3490     if (!$vga->{type}) {
3491         if ($arch eq 'aarch64') {
3492             $vga->{type} = 'virtio';
3493         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3494             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3495         } else {
3496             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3497         }
3498     }
3499
3500     # enable absolute mouse coordinates (needed by vnc)
3501     my $tablet;
3502     if (defined($conf->{tablet})) {
3503         $tablet = $conf->{tablet};
3504     } else {
3505         $tablet = $defaults->{tablet};
3506         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3507         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3508     }
3509
3510     if ($tablet) {
3511         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3512         my $kbd = print_keyboarddevice_full($conf, $arch);
3513         push @$devices, '-device', $kbd if defined($kbd);
3514     }
3515
3516     my $kvm_off = 0;
3517     my $gpu_passthrough;
3518
3519     # host pci devices
3520     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3521         my $d = parse_hostpci($conf->{"hostpci$i"});
3522         next if !$d;
3523
3524         my $pcie = $d->{pcie};
3525         if($pcie){
3526             die "q35 machine model is not enabled" if !$q35;
3527             $pciaddr = print_pcie_addr("hostpci$i");
3528         }else{
3529             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3530         }
3531
3532         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3533         my $romfile = $d->{romfile};
3534
3535         my $xvga = '';
3536         if ($d->{'x-vga'}) {
3537             $xvga = ',x-vga=on';
3538             $kvm_off = 1;
3539             $vga->{type} = 'none';
3540             $gpu_passthrough = 1;
3541
3542             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3543                 $xvga = "";
3544             }
3545         }
3546         my $pcidevices = $d->{pciid};
3547         my $multifunction = 1 if @$pcidevices > 1;
3548         my $sysfspath;
3549         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3550             my $id = $pcidevices->[0]->{id};
3551             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3552             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3553         } elsif ($d->{mdev}) {
3554             warn "ignoring mediated device with multifunction device\n";
3555         }
3556
3557         my $j=0;
3558         foreach my $pcidevice (@$pcidevices) {
3559
3560             my $id = "hostpci$i";
3561             $id .= ".$j" if $multifunction;
3562             my $addr = $pciaddr;
3563             $addr .= ".$j" if $multifunction;
3564             my $devicestr = "vfio-pci";
3565             if ($sysfspath) {
3566                 $devicestr .= ",sysfsdev=$sysfspath";
3567             } else {
3568                 $devicestr .= ",host=$pcidevice->{id}";
3569             }
3570             $devicestr .= ",id=$id$addr";
3571
3572             if($j == 0){
3573                 $devicestr .= "$rombar$xvga";
3574                 $devicestr .= ",multifunction=on" if $multifunction;
3575                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3576             }
3577
3578             push @$devices, '-device', $devicestr;
3579             $j++;
3580         }
3581     }
3582
3583     # usb devices
3584     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3585     push @$devices, @usbdevices if @usbdevices;
3586     # serial devices
3587     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3588         if (my $path = $conf->{"serial$i"}) {
3589             if ($path eq 'socket') {
3590                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3591                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3592                 # On aarch64, serial0 is the UART device. Qemu only allows
3593                 # connecting UART devices via the '-serial' command line, as
3594                 # the device has a fixed slot on the hardware...
3595                 if ($arch eq 'aarch64' && $i == 0) {
3596                     push @$devices, '-serial', "chardev:serial$i";
3597                 } else {
3598                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3599                 }
3600             } else {
3601                 die "no such serial device\n" if ! -c $path;
3602                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3603                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3604             }
3605         }
3606     }
3607
3608     # parallel devices
3609     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3610         if (my $path = $conf->{"parallel$i"}) {
3611             die "no such parallel device\n" if ! -c $path;
3612             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3613             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3614             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3615         }
3616     }
3617
3618
3619     my $sockets = 1;
3620     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3621     $sockets = $conf->{sockets} if  $conf->{sockets};
3622
3623     my $cores = $conf->{cores} || 1;
3624
3625     my $maxcpus = $sockets * $cores;
3626
3627     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3628
3629     my $allowed_vcpus = $cpuinfo->{cpus};
3630
3631     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3632         if ($allowed_vcpus < $maxcpus);
3633
3634     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3635
3636         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3637         for (my $i = 2; $i <= $vcpus; $i++)  {
3638             my $cpustr = print_cpu_device($conf,$i);
3639             push @$cmd, '-device', $cpustr;
3640         }
3641
3642     } else {
3643
3644         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3645     }
3646     push @$cmd, '-nodefaults';
3647
3648     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3649
3650     my $bootindex_hash = {};
3651     my $i = 1;
3652     foreach my $o (split(//, $bootorder)) {
3653         $bootindex_hash->{$o} = $i*100;
3654         $i++;
3655     }
3656
3657     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3658
3659     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3660
3661     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3662
3663     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3664         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3665         my $socket = vnc_socket($vmid);
3666         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3667     } else {
3668         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3669         push @$cmd, '-nographic';
3670     }
3671
3672     # time drift fix
3673     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3674
3675     my $useLocaltime = $conf->{localtime};
3676
3677     if ($winversion >= 5) { # windows
3678         $useLocaltime = 1 if !defined($conf->{localtime});
3679
3680         # use time drift fix when acpi is enabled
3681         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3682             $tdf = 1 if !defined($conf->{tdf});
3683         }
3684     }
3685
3686     if ($winversion >= 6) {
3687         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3688         push @$cmd, '-no-hpet';
3689     }
3690
3691     push @$rtcFlags, 'driftfix=slew' if $tdf;
3692
3693     if (!$kvm) {
3694         push @$machineFlags, 'accel=tcg';
3695     }
3696
3697     if ($machine_type) {
3698         push @$machineFlags, "type=${machine_type}";
3699     }
3700
3701     if ($conf->{startdate}) {
3702         push @$rtcFlags, "base=$conf->{startdate}";
3703     } elsif ($useLocaltime) {
3704         push @$rtcFlags, 'base=localtime';
3705     }
3706
3707     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3708
3709     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3710
3711     push @$cmd, '-S' if $conf->{freeze};
3712
3713     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3714
3715     # enable sound
3716     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3717     #push @$cmd, '-soundhw', 'es1370';
3718     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3719
3720     if (parse_guest_agent($conf)->{enabled}) {
3721         my $qgasocket = qmp_socket($vmid, 1);
3722         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3723         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3724         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3725         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3726     }
3727
3728     my $spice_port;
3729
3730     if ($qxlnum) {
3731         if ($qxlnum > 1) {
3732             if ($winversion){
3733                 for(my $i = 1; $i < $qxlnum; $i++){
3734                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3735                 }
3736             } else {
3737                 # assume other OS works like Linux
3738                 my ($ram, $vram) = ("134217728", "67108864");
3739                 if ($vga->{memory}) {
3740                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3741                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3742                 }
3743                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3744                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3745             }
3746         }
3747
3748         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3749
3750         my $nodename = PVE::INotify::nodename();
3751         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3752         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3753         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3754         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3755         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3756
3757         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3758
3759         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3760         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3761         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3762     }
3763
3764     # enable balloon by default, unless explicitly disabled
3765     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3766         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3767         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3768     }
3769
3770     if ($conf->{watchdog}) {
3771         my $wdopts = parse_watchdog($conf->{watchdog});
3772         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3773         my $watchdog = $wdopts->{model} || 'i6300esb';
3774         push @$devices, '-device', "$watchdog$pciaddr";
3775         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3776     }
3777
3778     my $vollist = [];
3779     my $scsicontroller = {};
3780     my $ahcicontroller = {};
3781     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3782
3783     # Add iscsi initiator name if available
3784     if (my $initiator = get_initiator_name()) {
3785         push @$devices, '-iscsi', "initiator-name=$initiator";
3786     }
3787
3788     foreach_drive($conf, sub {
3789         my ($ds, $drive) = @_;
3790
3791         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3792             push @$vollist, $drive->{file};
3793         }
3794
3795         # ignore efidisk here, already added in bios/fw handling code above
3796         return if $drive->{interface} eq 'efidisk';
3797
3798         $use_virtio = 1 if $ds =~ m/^virtio/;
3799
3800         if (drive_is_cdrom ($drive)) {
3801             if ($bootindex_hash->{d}) {
3802                 $drive->{bootindex} = $bootindex_hash->{d};
3803                 $bootindex_hash->{d} += 1;
3804             }
3805         } else {
3806             if ($bootindex_hash->{c}) {
3807                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3808                 $bootindex_hash->{c} += 1;
3809             }
3810         }
3811
3812         if($drive->{interface} eq 'virtio'){
3813            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3814         }
3815
3816         if ($drive->{interface} eq 'scsi') {
3817
3818             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3819
3820             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3821             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3822
3823             my $iothread = '';
3824             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3825                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3826                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3827             } elsif ($drive->{iothread}) {
3828                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3829             }
3830
3831             my $queues = '';
3832             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3833                 $queues = ",num_queues=$drive->{queues}";
3834             }
3835
3836             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3837             $scsicontroller->{$controller}=1;
3838         }
3839
3840         if ($drive->{interface} eq 'sata') {
3841            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3842            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3843            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3844            $ahcicontroller->{$controller}=1;
3845         }
3846
3847         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3848         push @$devices, '-drive',$drive_cmd;
3849         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3850     });
3851
3852     for (my $i = 0; $i < $MAX_NETS; $i++) {
3853          next if !$conf->{"net$i"};
3854          my $d = parse_net($conf->{"net$i"});
3855          next if !$d;
3856
3857          $use_virtio = 1 if $d->{model} eq 'virtio';
3858
3859          if ($bootindex_hash->{n}) {
3860             $d->{bootindex} = $bootindex_hash->{n};
3861             $bootindex_hash->{n} += 1;
3862          }
3863
3864          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3865          push @$devices, '-netdev', $netdevfull;
3866
3867          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3868          push @$devices, '-device', $netdevicefull;
3869     }
3870
3871     if (!$q35) {
3872         # add pci bridges
3873         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3874            $bridges->{1} = 1;
3875            $bridges->{2} = 1;
3876         }
3877
3878         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3879
3880         while (my ($k, $v) = each %$bridges) {
3881             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3882             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3883         }
3884     }
3885
3886     # add custom args
3887     if ($conf->{args}) {
3888         my $aa = PVE::Tools::split_args($conf->{args});
3889         push @$cmd, @$aa;
3890     }
3891
3892     push @$cmd, @$devices;
3893     push @$cmd, '-rtc', join(',', @$rtcFlags)
3894         if scalar(@$rtcFlags);
3895     push @$cmd, '-machine', join(',', @$machineFlags)
3896         if scalar(@$machineFlags);
3897     push @$cmd, '-global', join(',', @$globalFlags)
3898         if scalar(@$globalFlags);
3899
3900     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3901 }
3902
3903 sub vnc_socket {
3904     my ($vmid) = @_;
3905     return "${var_run_tmpdir}/$vmid.vnc";
3906 }
3907
3908 sub spice_port {
3909     my ($vmid) = @_;
3910
3911     my $res = vm_mon_cmd($vmid, 'query-spice');
3912
3913     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3914 }
3915
3916 sub qmp_socket {
3917     my ($vmid, $qga, $name) = @_;
3918     my $sockettype = $qga ? 'qga' : 'qmp';
3919     my $ext = $name ? '-'.$name : '';
3920     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3921 }
3922
3923 sub pidfile_name {
3924     my ($vmid) = @_;
3925     return "${var_run_tmpdir}/$vmid.pid";
3926 }
3927
3928 sub vm_devices_list {
3929     my ($vmid) = @_;
3930
3931     my $res = vm_mon_cmd($vmid, 'query-pci');
3932     my $devices_to_check = [];
3933     my $devices = {};
3934     foreach my $pcibus (@$res) {
3935         push @$devices_to_check, @{$pcibus->{devices}},
3936     }
3937
3938     while (@$devices_to_check) {
3939         my $to_check = [];
3940         for my $d (@$devices_to_check) {
3941             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3942             next if !$d->{'pci_bridge'};
3943
3944             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3945             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3946         }
3947         $devices_to_check = $to_check;
3948     }
3949
3950     my $resblock = vm_mon_cmd($vmid, 'query-block');
3951     foreach my $block (@$resblock) {
3952         if($block->{device} =~ m/^drive-(\S+)/){
3953                 $devices->{$1} = 1;
3954         }
3955     }
3956
3957     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3958     foreach my $mice (@$resmice) {
3959         if ($mice->{name} eq 'QEMU HID Tablet') {
3960             $devices->{tablet} = 1;
3961             last;
3962         }
3963     }
3964
3965     # for usb devices there is no query-usb
3966     # but we can iterate over the entries in
3967     # qom-list path=/machine/peripheral
3968     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3969     foreach my $per (@$resperipheral) {
3970         if ($per->{name} =~ m/^usb\d+$/) {
3971             $devices->{$per->{name}} = 1;
3972         }
3973     }
3974
3975     return $devices;
3976 }
3977
3978 sub vm_deviceplug {
3979     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3980
3981     my $q35 = machine_type_is_q35($conf);
3982
3983     my $devices_list = vm_devices_list($vmid);
3984     return 1 if defined($devices_list->{$deviceid});
3985
3986     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3987
3988     if ($deviceid eq 'tablet') {
3989
3990         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3991
3992     } elsif ($deviceid eq 'keyboard') {
3993
3994         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3995
3996     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3997
3998         die "usb hotplug currently not reliable\n";
3999         # since we can't reliably hot unplug all added usb devices
4000         # and usb passthrough disables live migration
4001         # we disable usb hotplugging for now
4002         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4003
4004     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4005
4006         qemu_iothread_add($vmid, $deviceid, $device);
4007
4008         qemu_driveadd($storecfg, $vmid, $device);
4009         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4010
4011         qemu_deviceadd($vmid, $devicefull);
4012         eval { qemu_deviceaddverify($vmid, $deviceid); };
4013         if (my $err = $@) {
4014             eval { qemu_drivedel($vmid, $deviceid); };
4015             warn $@ if $@;
4016             die $err;
4017         }
4018
4019     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4020
4021
4022         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4023         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4024         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4025
4026         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4027
4028         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4029             qemu_iothread_add($vmid, $deviceid, $device);
4030             $devicefull .= ",iothread=iothread-$deviceid";
4031         }
4032
4033         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4034             $devicefull .= ",num_queues=$device->{queues}";
4035         }
4036
4037         qemu_deviceadd($vmid, $devicefull);
4038         qemu_deviceaddverify($vmid, $deviceid);
4039
4040     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4041
4042         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4043         qemu_driveadd($storecfg, $vmid, $device);
4044
4045         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4046         eval { qemu_deviceadd($vmid, $devicefull); };
4047         if (my $err = $@) {
4048             eval { qemu_drivedel($vmid, $deviceid); };
4049             warn $@ if $@;
4050             die $err;
4051         }
4052
4053     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4054
4055         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4056
4057         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4058         my $use_old_bios_files = undef;
4059         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4060
4061         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4062         qemu_deviceadd($vmid, $netdevicefull);
4063         eval { qemu_deviceaddverify($vmid, $deviceid); };
4064         if (my $err = $@) {
4065             eval { qemu_netdevdel($vmid, $deviceid); };
4066             warn $@ if $@;
4067             die $err;
4068         }
4069
4070     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4071
4072         my $bridgeid = $2;
4073         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4074         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4075
4076         qemu_deviceadd($vmid, $devicefull);
4077         qemu_deviceaddverify($vmid, $deviceid);
4078
4079     } else {
4080         die "can't hotplug device '$deviceid'\n";
4081     }
4082
4083     return 1;
4084 }
4085
4086 # fixme: this should raise exceptions on error!
4087 sub vm_deviceunplug {
4088     my ($vmid, $conf, $deviceid) = @_;
4089
4090     my $devices_list = vm_devices_list($vmid);
4091     return 1 if !defined($devices_list->{$deviceid});
4092
4093     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4094
4095     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4096
4097         qemu_devicedel($vmid, $deviceid);
4098
4099     } elsif ($deviceid =~ m/^usb\d+$/) {
4100
4101         die "usb hotplug currently not reliable\n";
4102         # when unplugging usb devices this way,
4103         # there may be remaining usb controllers/hubs
4104         # so we disable it for now
4105         qemu_devicedel($vmid, $deviceid);
4106         qemu_devicedelverify($vmid, $deviceid);
4107
4108     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4109
4110         qemu_devicedel($vmid, $deviceid);
4111         qemu_devicedelverify($vmid, $deviceid);
4112         qemu_drivedel($vmid, $deviceid);
4113         qemu_iothread_del($conf, $vmid, $deviceid);
4114
4115     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4116
4117         qemu_devicedel($vmid, $deviceid);
4118         qemu_devicedelverify($vmid, $deviceid);
4119         qemu_iothread_del($conf, $vmid, $deviceid);
4120
4121     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4122
4123         qemu_devicedel($vmid, $deviceid);
4124         qemu_drivedel($vmid, $deviceid);
4125         qemu_deletescsihw($conf, $vmid, $deviceid);
4126
4127     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4128
4129         qemu_devicedel($vmid, $deviceid);
4130         qemu_devicedelverify($vmid, $deviceid);
4131         qemu_netdevdel($vmid, $deviceid);
4132
4133     } else {
4134         die "can't unplug device '$deviceid'\n";
4135     }
4136
4137     return 1;
4138 }
4139
4140 sub qemu_deviceadd {
4141     my ($vmid, $devicefull) = @_;
4142
4143     $devicefull = "driver=".$devicefull;
4144     my %options =  split(/[=,]/, $devicefull);
4145
4146     vm_mon_cmd($vmid, "device_add" , %options);
4147 }
4148
4149 sub qemu_devicedel {
4150     my ($vmid, $deviceid) = @_;
4151
4152     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4153 }
4154
4155 sub qemu_iothread_add {
4156     my($vmid, $deviceid, $device) = @_;
4157
4158     if ($device->{iothread}) {
4159         my $iothreads = vm_iothreads_list($vmid);
4160         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4161     }
4162 }
4163
4164 sub qemu_iothread_del {
4165     my($conf, $vmid, $deviceid) = @_;
4166
4167     my $device = parse_drive($deviceid, $conf->{$deviceid});
4168     if ($device->{iothread}) {
4169         my $iothreads = vm_iothreads_list($vmid);
4170         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4171     }
4172 }
4173
4174 sub qemu_objectadd {
4175     my($vmid, $objectid, $qomtype) = @_;
4176
4177     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4178
4179     return 1;
4180 }
4181
4182 sub qemu_objectdel {
4183     my($vmid, $objectid) = @_;
4184
4185     vm_mon_cmd($vmid, "object-del", id => $objectid);
4186
4187     return 1;
4188 }
4189
4190 sub qemu_driveadd {
4191     my ($storecfg, $vmid, $device) = @_;
4192
4193     my $drive = print_drive_full($storecfg, $vmid, $device);
4194     $drive =~ s/\\/\\\\/g;
4195     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4196
4197     # If the command succeeds qemu prints: "OK"
4198     return 1 if $ret =~ m/OK/s;
4199
4200     die "adding drive failed: $ret\n";
4201 }
4202
4203 sub qemu_drivedel {
4204     my($vmid, $deviceid) = @_;
4205
4206     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4207     $ret =~ s/^\s+//;
4208
4209     return 1 if $ret eq "";
4210
4211     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4212     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4213
4214     die "deleting drive $deviceid failed : $ret\n";
4215 }
4216
4217 sub qemu_deviceaddverify {
4218     my ($vmid, $deviceid) = @_;
4219
4220     for (my $i = 0; $i <= 5; $i++) {
4221          my $devices_list = vm_devices_list($vmid);
4222          return 1 if defined($devices_list->{$deviceid});
4223          sleep 1;
4224     }
4225
4226     die "error on hotplug device '$deviceid'\n";
4227 }
4228
4229
4230 sub qemu_devicedelverify {
4231     my ($vmid, $deviceid) = @_;
4232
4233     # need to verify that the device is correctly removed as device_del
4234     # is async and empty return is not reliable
4235
4236     for (my $i = 0; $i <= 5; $i++) {
4237          my $devices_list = vm_devices_list($vmid);
4238          return 1 if !defined($devices_list->{$deviceid});
4239          sleep 1;
4240     }
4241
4242     die "error on hot-unplugging device '$deviceid'\n";
4243 }
4244
4245 sub qemu_findorcreatescsihw {
4246     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4247
4248     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4249
4250     my $scsihwid="$controller_prefix$controller";
4251     my $devices_list = vm_devices_list($vmid);
4252
4253     if(!defined($devices_list->{$scsihwid})) {
4254         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4255     }
4256
4257     return 1;
4258 }
4259
4260 sub qemu_deletescsihw {
4261     my ($conf, $vmid, $opt) = @_;
4262
4263     my $device = parse_drive($opt, $conf->{$opt});
4264
4265     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4266         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4267         return 1;
4268     }
4269
4270     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4271
4272     my $devices_list = vm_devices_list($vmid);
4273     foreach my $opt (keys %{$devices_list}) {
4274         if (PVE::QemuServer::is_valid_drivename($opt)) {
4275             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4276             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4277                 return 1;
4278             }
4279         }
4280     }
4281
4282     my $scsihwid="scsihw$controller";
4283
4284     vm_deviceunplug($vmid, $conf, $scsihwid);
4285
4286     return 1;
4287 }
4288
4289 sub qemu_add_pci_bridge {
4290     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4291
4292     my $bridges = {};
4293
4294     my $bridgeid;
4295
4296     print_pci_addr($device, $bridges, $arch, $machine_type);
4297
4298     while (my ($k, $v) = each %$bridges) {
4299         $bridgeid = $k;
4300     }
4301     return 1 if !defined($bridgeid) || $bridgeid < 1;
4302
4303     my $bridge = "pci.$bridgeid";
4304     my $devices_list = vm_devices_list($vmid);
4305
4306     if (!defined($devices_list->{$bridge})) {
4307         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4308     }
4309
4310     return 1;
4311 }
4312
4313 sub qemu_set_link_status {
4314     my ($vmid, $device, $up) = @_;
4315
4316     vm_mon_cmd($vmid, "set_link", name => $device,
4317                up => $up ? JSON::true : JSON::false);
4318 }
4319
4320 sub qemu_netdevadd {
4321     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4322
4323     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4324     my %options =  split(/[=,]/, $netdev);
4325
4326     vm_mon_cmd($vmid, "netdev_add",  %options);
4327     return 1;
4328 }
4329
4330 sub qemu_netdevdel {
4331     my ($vmid, $deviceid) = @_;
4332
4333     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4334 }
4335
4336 sub qemu_usb_hotplug {
4337     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4338
4339     return if !$device;
4340
4341     # remove the old one first
4342     vm_deviceunplug($vmid, $conf, $deviceid);
4343
4344     # check if xhci controller is necessary and available
4345     if ($device->{usb3}) {
4346
4347         my $devicelist = vm_devices_list($vmid);
4348
4349         if (!$devicelist->{xhci}) {
4350             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4351             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4352         }
4353     }
4354     my $d = parse_usb_device($device->{host});
4355     $d->{usb3} = $device->{usb3};
4356
4357     # add the new one
4358     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4359 }
4360
4361 sub qemu_cpu_hotplug {
4362     my ($vmid, $conf, $vcpus) = @_;
4363
4364     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4365
4366     my $sockets = 1;
4367     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4368     $sockets = $conf->{sockets} if  $conf->{sockets};
4369     my $cores = $conf->{cores} || 1;
4370     my $maxcpus = $sockets * $cores;
4371
4372     $vcpus = $maxcpus if !$vcpus;
4373
4374     die "you can't add more vcpus than maxcpus\n"
4375         if $vcpus > $maxcpus;
4376
4377     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4378
4379     if ($vcpus < $currentvcpus) {
4380
4381         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4382
4383             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4384                 qemu_devicedel($vmid, "cpu$i");
4385                 my $retry = 0;
4386                 my $currentrunningvcpus = undef;
4387                 while (1) {
4388                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4389                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4390                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4391                     $retry++;
4392                     sleep 1;
4393                 }
4394                 #update conf after each succesfull cpu unplug
4395                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4396                 PVE::QemuConfig->write_config($vmid, $conf);
4397             }
4398         } else {
4399             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4400         }
4401
4402         return;
4403     }
4404
4405     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4406     die "vcpus in running vm does not match its configuration\n"
4407         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4408
4409     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4410
4411         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4412             my $cpustr = print_cpu_device($conf, $i);
4413             qemu_deviceadd($vmid, $cpustr);
4414
4415             my $retry = 0;
4416             my $currentrunningvcpus = undef;
4417             while (1) {
4418                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4419                 last if scalar(@{$currentrunningvcpus}) == $i;
4420                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4421                 sleep 1;
4422                 $retry++;
4423             }
4424             #update conf after each succesfull cpu hotplug
4425             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4426             PVE::QemuConfig->write_config($vmid, $conf);
4427         }
4428     } else {
4429
4430         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4431             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4432         }
4433     }
4434 }
4435
4436 sub qemu_block_set_io_throttle {
4437     my ($vmid, $deviceid,
4438         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4439         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4440         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4441         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4442
4443     return if !check_running($vmid) ;
4444
4445     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4446         bps => int($bps),
4447         bps_rd => int($bps_rd),
4448         bps_wr => int($bps_wr),
4449         iops => int($iops),
4450         iops_rd => int($iops_rd),
4451         iops_wr => int($iops_wr),
4452         bps_max => int($bps_max),
4453         bps_rd_max => int($bps_rd_max),
4454         bps_wr_max => int($bps_wr_max),
4455         iops_max => int($iops_max),
4456         iops_rd_max => int($iops_rd_max),
4457         iops_wr_max => int($iops_wr_max),
4458         bps_max_length => int($bps_max_length),
4459         bps_rd_max_length => int($bps_rd_max_length),
4460         bps_wr_max_length => int($bps_wr_max_length),
4461         iops_max_length => int($iops_max_length),
4462         iops_rd_max_length => int($iops_rd_max_length),
4463         iops_wr_max_length => int($iops_wr_max_length),
4464     );
4465
4466 }
4467
4468 # old code, only used to shutdown old VM after update
4469 sub __read_avail {
4470     my ($fh, $timeout) = @_;
4471
4472     my $sel = new IO::Select;
4473     $sel->add($fh);
4474
4475     my $res = '';
4476     my $buf;
4477
4478     my @ready;
4479     while (scalar (@ready = $sel->can_read($timeout))) {
4480         my $count;
4481         if ($count = $fh->sysread($buf, 8192)) {
4482             if ($buf =~ /^(.*)\(qemu\) $/s) {
4483                 $res .= $1;
4484                 last;
4485             } else {
4486                 $res .= $buf;
4487             }
4488         } else {
4489             if (!defined($count)) {
4490                 die "$!\n";
4491             }
4492             last;
4493         }
4494     }
4495
4496     die "monitor read timeout\n" if !scalar(@ready);
4497
4498     return $res;
4499 }
4500
4501 sub qemu_block_resize {
4502     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4503
4504     my $running = check_running($vmid);
4505
4506     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4507
4508     return if !$running;
4509
4510     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4511
4512 }
4513
4514 sub qemu_volume_snapshot {
4515     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4516
4517     my $running = check_running($vmid);
4518
4519     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4520         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4521     } else {
4522         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4523     }
4524 }
4525
4526 sub qemu_volume_snapshot_delete {
4527     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4528
4529     my $running = check_running($vmid);
4530
4531     if($running) {
4532
4533         $running = undef;
4534         my $conf = PVE::QemuConfig->load_config($vmid);
4535         foreach_drive($conf, sub {
4536             my ($ds, $drive) = @_;
4537             $running = 1 if $drive->{file} eq $volid;
4538         });
4539     }
4540
4541     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4542         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4543     } else {
4544         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4545     }
4546 }
4547
4548 sub set_migration_caps {
4549     my ($vmid) = @_;
4550
4551     my $cap_ref = [];
4552
4553     my $enabled_cap = {
4554         "auto-converge" => 1,
4555         "xbzrle" => 1,
4556         "x-rdma-pin-all" => 0,
4557         "zero-blocks" => 0,
4558         "compress" => 0
4559     };
4560
4561     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4562
4563     for my $supported_capability (@$supported_capabilities) {
4564         push @$cap_ref, {
4565             capability => $supported_capability->{capability},
4566             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4567         };
4568     }
4569
4570     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4571 }
4572
4573 my $fast_plug_option = {
4574     'lock' => 1,
4575     'name' => 1,
4576     'onboot' => 1,
4577     'shares' => 1,
4578     'startup' => 1,
4579     'description' => 1,
4580     'protection' => 1,
4581     'vmstatestorage' => 1,
4582 };
4583
4584 # hotplug changes in [PENDING]
4585 # $selection hash can be used to only apply specified options, for
4586 # example: { cores => 1 } (only apply changed 'cores')
4587 # $errors ref is used to return error messages
4588 sub vmconfig_hotplug_pending {
4589     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4590
4591     my $defaults = load_defaults();
4592     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4593
4594     # commit values which do not have any impact on running VM first
4595     # Note: those option cannot raise errors, we we do not care about
4596     # $selection and always apply them.
4597
4598     my $add_error = sub {
4599         my ($opt, $msg) = @_;
4600         $errors->{$opt} = "hotplug problem - $msg";
4601     };
4602
4603     my $changes = 0;
4604     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4605         if ($fast_plug_option->{$opt}) {
4606             $conf->{$opt} = $conf->{pending}->{$opt};
4607             delete $conf->{pending}->{$opt};
4608             $changes = 1;
4609         }
4610     }
4611
4612     if ($changes) {
4613         PVE::QemuConfig->write_config($vmid, $conf);
4614         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4615     }
4616
4617     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4618
4619     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4620     while (my ($opt, $force) = each %$pending_delete_hash) {
4621         next if $selection && !$selection->{$opt};
4622         eval {
4623             if ($opt eq 'hotplug') {
4624                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4625             } elsif ($opt eq 'tablet') {
4626                 die "skip\n" if !$hotplug_features->{usb};
4627                 if ($defaults->{tablet}) {
4628                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4629                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4630                         if $arch eq 'aarch64';
4631                 } else {
4632                     vm_deviceunplug($vmid, $conf, 'tablet');
4633                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4634                 }
4635             } elsif ($opt =~ m/^usb\d+/) {
4636                 die "skip\n";
4637                 # since we cannot reliably hot unplug usb devices
4638                 # we are disabling it
4639                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4640                 vm_deviceunplug($vmid, $conf, $opt);
4641             } elsif ($opt eq 'vcpus') {
4642                 die "skip\n" if !$hotplug_features->{cpu};
4643                 qemu_cpu_hotplug($vmid, $conf, undef);
4644             } elsif ($opt eq 'balloon') {
4645                 # enable balloon device is not hotpluggable
4646                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4647                 # here we reset the ballooning value to memory
4648                 my $balloon = $conf->{memory} || $defaults->{memory};
4649                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4650             } elsif ($fast_plug_option->{$opt}) {
4651                 # do nothing
4652             } elsif ($opt =~ m/^net(\d+)$/) {
4653                 die "skip\n" if !$hotplug_features->{network};
4654                 vm_deviceunplug($vmid, $conf, $opt);
4655             } elsif (is_valid_drivename($opt)) {
4656                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4657                 vm_deviceunplug($vmid, $conf, $opt);
4658                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4659             } elsif ($opt =~ m/^memory$/) {
4660                 die "skip\n" if !$hotplug_features->{memory};
4661                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4662             } elsif ($opt eq 'cpuunits') {
4663                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4664             } elsif ($opt eq 'cpulimit') {
4665                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4666             } else {
4667                 die "skip\n";
4668             }
4669         };
4670         if (my $err = $@) {
4671             &$add_error($opt, $err) if $err ne "skip\n";
4672         } else {
4673             # save new config if hotplug was successful
4674             delete $conf->{$opt};
4675             vmconfig_undelete_pending_option($conf, $opt);
4676             PVE::QemuConfig->write_config($vmid, $conf);
4677             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4678         }
4679     }
4680
4681     my $apply_pending_cloudinit;
4682     $apply_pending_cloudinit = sub {
4683         my ($key, $value) = @_;
4684         $apply_pending_cloudinit = sub {}; # once is enough
4685
4686         my @cloudinit_opts = keys %$confdesc_cloudinit;
4687         foreach my $opt (keys %{$conf->{pending}}) {
4688             next if !grep { $_ eq $opt } @cloudinit_opts;
4689             $conf->{$opt} = delete $conf->{pending}->{$opt};
4690         }
4691
4692         my $new_conf = { %$conf };
4693         $new_conf->{$key} = $value;
4694         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4695     };
4696
4697     foreach my $opt (keys %{$conf->{pending}}) {
4698         next if $selection && !$selection->{$opt};
4699         my $value = $conf->{pending}->{$opt};
4700         eval {
4701             if ($opt eq 'hotplug') {
4702                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4703             } elsif ($opt eq 'tablet') {
4704                 die "skip\n" if !$hotplug_features->{usb};
4705                 if ($value == 1) {
4706                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4707                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4708                         if $arch eq 'aarch64';
4709                 } elsif ($value == 0) {
4710                     vm_deviceunplug($vmid, $conf, 'tablet');
4711                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4712                 }
4713             } elsif ($opt =~ m/^usb\d+$/) {
4714                 die "skip\n";
4715                 # since we cannot reliably hot unplug usb devices
4716                 # we are disabling it
4717                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4718                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4719                 die "skip\n" if !$d;
4720                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4721             } elsif ($opt eq 'vcpus') {
4722                 die "skip\n" if !$hotplug_features->{cpu};
4723                 qemu_cpu_hotplug($vmid, $conf, $value);
4724             } elsif ($opt eq 'balloon') {
4725                 # enable/disable balloning device is not hotpluggable
4726                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4727                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4728                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4729
4730                 # allow manual ballooning if shares is set to zero
4731                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4732                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4733                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4734                 }
4735             } elsif ($opt =~ m/^net(\d+)$/) {
4736                 # some changes can be done without hotplug
4737                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4738                                     $vmid, $opt, $value, $arch, $machine_type);
4739             } elsif (is_valid_drivename($opt)) {
4740                 # some changes can be done without hotplug
4741                 my $drive = parse_drive($opt, $value);
4742                 if (drive_is_cloudinit($drive)) {
4743                     &$apply_pending_cloudinit($opt, $value);
4744                 }
4745                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4746                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4747             } elsif ($opt =~ m/^memory$/) { #dimms
4748                 die "skip\n" if !$hotplug_features->{memory};
4749                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4750             } elsif ($opt eq 'cpuunits') {
4751                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4752             } elsif ($opt eq 'cpulimit') {
4753                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4754                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4755             } else {
4756                 die "skip\n";  # skip non-hot-pluggable options
4757             }
4758         };
4759         if (my $err = $@) {
4760             &$add_error($opt, $err) if $err ne "skip\n";
4761         } else {
4762             # save new config if hotplug was successful
4763             $conf->{$opt} = $value;
4764             delete $conf->{pending}->{$opt};
4765             PVE::QemuConfig->write_config($vmid, $conf);
4766             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4767         }
4768     }
4769 }
4770
4771 sub try_deallocate_drive {
4772     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4773
4774     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4775         my $volid = $drive->{file};
4776         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4777             my $sid = PVE::Storage::parse_volume_id($volid);
4778             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4779
4780             # check if the disk is really unused
4781             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4782                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4783             PVE::Storage::vdisk_free($storecfg, $volid);
4784             return 1;
4785         } else {
4786             # If vm is not owner of this disk remove from config
4787             return 1;
4788         }
4789     }
4790
4791     return undef;
4792 }
4793
4794 sub vmconfig_delete_or_detach_drive {
4795     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4796
4797     my $drive = parse_drive($opt, $conf->{$opt});
4798
4799     my $rpcenv = PVE::RPCEnvironment::get();
4800     my $authuser = $rpcenv->get_user();
4801
4802     if ($force) {
4803         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4804         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4805     } else {
4806         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4807     }
4808 }
4809
4810 sub vmconfig_apply_pending {
4811     my ($vmid, $conf, $storecfg) = @_;
4812
4813     # cold plug
4814
4815     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4816     while (my ($opt, $force) = each %$pending_delete_hash) {
4817         die "internal error" if $opt =~ m/^unused/;
4818         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4819         if (!defined($conf->{$opt})) {
4820             vmconfig_undelete_pending_option($conf, $opt);
4821             PVE::QemuConfig->write_config($vmid, $conf);
4822         } elsif (is_valid_drivename($opt)) {
4823             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4824             vmconfig_undelete_pending_option($conf, $opt);
4825             delete $conf->{$opt};
4826             PVE::QemuConfig->write_config($vmid, $conf);
4827         } else {
4828             vmconfig_undelete_pending_option($conf, $opt);
4829             delete $conf->{$opt};
4830             PVE::QemuConfig->write_config($vmid, $conf);
4831         }
4832     }
4833
4834     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4835
4836     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4837         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4838
4839         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4840             # skip if nothing changed
4841         } elsif (is_valid_drivename($opt)) {
4842             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4843                 if defined($conf->{$opt});
4844             $conf->{$opt} = $conf->{pending}->{$opt};
4845         } else {
4846             $conf->{$opt} = $conf->{pending}->{$opt};
4847         }
4848
4849         delete $conf->{pending}->{$opt};
4850         PVE::QemuConfig->write_config($vmid, $conf);
4851     }
4852 }
4853
4854 my $safe_num_ne = sub {
4855     my ($a, $b) = @_;
4856
4857     return 0 if !defined($a) && !defined($b);
4858     return 1 if !defined($a);
4859     return 1 if !defined($b);
4860
4861     return $a != $b;
4862 };
4863
4864 my $safe_string_ne = sub {
4865     my ($a, $b) = @_;
4866
4867     return 0 if !defined($a) && !defined($b);
4868     return 1 if !defined($a);
4869     return 1 if !defined($b);
4870
4871     return $a ne $b;
4872 };
4873
4874 sub vmconfig_update_net {
4875     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4876
4877     my $newnet = parse_net($value);
4878
4879     if ($conf->{$opt}) {
4880         my $oldnet = parse_net($conf->{$opt});
4881
4882         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4883             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4884             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4885             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4886
4887             # for non online change, we try to hot-unplug
4888             die "skip\n" if !$hotplug;
4889             vm_deviceunplug($vmid, $conf, $opt);
4890         } else {
4891
4892             die "internal error" if $opt !~ m/net(\d+)/;
4893             my $iface = "tap${vmid}i$1";
4894
4895             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4896                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4897                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4898                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4899                 PVE::Network::tap_unplug($iface);
4900                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4901             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4902                 # Rate can be applied on its own but any change above needs to
4903                 # include the rate in tap_plug since OVS resets everything.
4904                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4905             }
4906
4907             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4908                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4909             }
4910
4911             return 1;
4912         }
4913     }
4914
4915     if ($hotplug) {
4916         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4917     } else {
4918         die "skip\n";
4919     }
4920 }
4921
4922 sub vmconfig_update_disk {
4923     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4924
4925     # fixme: do we need force?
4926
4927     my $drive = parse_drive($opt, $value);
4928
4929     if ($conf->{$opt}) {
4930
4931         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4932
4933             my $media = $drive->{media} || 'disk';
4934             my $oldmedia = $old_drive->{media} || 'disk';
4935             die "unable to change media type\n" if $media ne $oldmedia;
4936
4937             if (!drive_is_cdrom($old_drive)) {
4938
4939                 if ($drive->{file} ne $old_drive->{file}) {
4940
4941                     die "skip\n" if !$hotplug;
4942
4943                     # unplug and register as unused
4944                     vm_deviceunplug($vmid, $conf, $opt);
4945                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4946
4947                 } else {
4948                     # update existing disk
4949
4950                     # skip non hotpluggable value
4951                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4952                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4953                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4954                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4955                         die "skip\n";
4956                     }
4957
4958                     # apply throttle
4959                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4960                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4961                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4962                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4963                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4964                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4965                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4966                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4967                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4968                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4969                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4970                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4971                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4972                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4973                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4974                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4975                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4976                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4977
4978                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4979                                                    ($drive->{mbps} || 0)*1024*1024,
4980                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4981                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4982                                                    $drive->{iops} || 0,
4983                                                    $drive->{iops_rd} || 0,
4984                                                    $drive->{iops_wr} || 0,
4985                                                    ($drive->{mbps_max} || 0)*1024*1024,
4986                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4987                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4988                                                    $drive->{iops_max} || 0,
4989                                                    $drive->{iops_rd_max} || 0,
4990                                                    $drive->{iops_wr_max} || 0,
4991                                                    $drive->{bps_max_length} || 1,
4992                                                    $drive->{bps_rd_max_length} || 1,
4993                                                    $drive->{bps_wr_max_length} || 1,
4994                                                    $drive->{iops_max_length} || 1,
4995                                                    $drive->{iops_rd_max_length} || 1,
4996                                                    $drive->{iops_wr_max_length} || 1);
4997
4998                     }
4999
5000                     return 1;
5001                 }
5002
5003             } else { # cdrom
5004
5005                 if ($drive->{file} eq 'none') {
5006                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5007                     if (drive_is_cloudinit($old_drive)) {
5008                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5009                     }
5010                 } else {
5011                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5012                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5013                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5014                 }
5015
5016                 return 1;
5017             }
5018         }
5019     }
5020
5021     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5022     # hotplug new disks
5023     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5024     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5025 }
5026
5027 sub vm_start {
5028     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5029         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5030
5031     PVE::QemuConfig->lock_config($vmid, sub {
5032         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5033
5034         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5035
5036         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5037
5038         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5039
5040         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5041             vmconfig_apply_pending($vmid, $conf, $storecfg);
5042             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5043         }
5044
5045         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5046
5047         my $defaults = load_defaults();
5048
5049         # set environment variable useful inside network script
5050         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5051
5052         my $local_volumes = {};
5053
5054         if ($targetstorage) {
5055             foreach_drive($conf, sub {
5056                 my ($ds, $drive) = @_;
5057
5058                 return if drive_is_cdrom($drive);
5059
5060                 my $volid = $drive->{file};
5061
5062                 return if !$volid;
5063
5064                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5065
5066                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5067                 return if $scfg->{shared};
5068                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5069             });
5070
5071             my $format = undef;
5072
5073             foreach my $opt (sort keys %$local_volumes) {
5074
5075                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5076                 my $drive = parse_drive($opt, $conf->{$opt});
5077
5078                 #if remote storage is specified, use default format
5079                 if ($targetstorage && $targetstorage ne "1") {
5080                     $storeid = $targetstorage;
5081                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5082                     $format = $defFormat;
5083                 } else {
5084                     #else we use same format than original
5085                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5086                     $format = qemu_img_format($scfg, $volid);
5087                 }
5088
5089                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5090                 my $newdrive = $drive;
5091                 $newdrive->{format} = $format;
5092                 $newdrive->{file} = $newvolid;
5093                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5094                 $local_volumes->{$opt} = $drivestr;
5095                 #pass drive to conf for command line
5096                 $conf->{$opt} = $drivestr;
5097             }
5098         }
5099
5100         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5101
5102         my $migrate_port = 0;
5103         my $migrate_uri;
5104         if ($statefile) {
5105             if ($statefile eq 'tcp') {
5106                 my $localip = "localhost";
5107                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5108                 my $nodename = PVE::INotify::nodename();
5109
5110                 if (!defined($migration_type)) {
5111                     if (defined($datacenterconf->{migration}->{type})) {
5112                         $migration_type = $datacenterconf->{migration}->{type};
5113                     } else {
5114                         $migration_type = 'secure';
5115                     }
5116                 }
5117
5118                 if ($migration_type eq 'insecure') {
5119                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5120                     if ($migrate_network_addr) {
5121                         $localip = $migrate_network_addr;
5122                     } else {
5123                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5124                     }
5125
5126                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5127                 }
5128
5129                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5130                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5131                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5132                 push @$cmd, '-incoming', $migrate_uri;
5133                 push @$cmd, '-S';
5134
5135             } elsif ($statefile eq 'unix') {
5136                 # should be default for secure migrations as a ssh TCP forward
5137                 # tunnel is not deterministic reliable ready and fails regurarly
5138                 # to set up in time, so use UNIX socket forwards
5139                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5140                 unlink $socket_addr;
5141
5142                 $migrate_uri = "unix:$socket_addr";
5143
5144                 push @$cmd, '-incoming', $migrate_uri;
5145                 push @$cmd, '-S';
5146
5147             } else {
5148                 push @$cmd, '-loadstate', $statefile;
5149             }
5150         } elsif ($paused) {
5151             push @$cmd, '-S';
5152         }
5153
5154         # host pci devices
5155         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5156           my $d = parse_hostpci($conf->{"hostpci$i"});
5157           next if !$d;
5158           my $pcidevices = $d->{pciid};
5159           foreach my $pcidevice (@$pcidevices) {
5160                 my $pciid = $pcidevice->{id};
5161
5162                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5163                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5164                 die "no pci device info for device '$pciid'\n" if !$info;
5165
5166                 if ($d->{mdev}) {
5167                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5168                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5169                 } else {
5170                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5171                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5172                     die "can't reset pci device '$pciid'\n"
5173                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5174                 }
5175           }
5176         }
5177
5178         PVE::Storage::activate_volumes($storecfg, $vollist);
5179
5180         if (!check_running($vmid, 1)) {
5181             eval {
5182                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5183                     outfunc => sub {}, errfunc => sub {});
5184             };
5185         }
5186
5187         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5188                                                   : $defaults->{cpuunits};
5189
5190         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5191         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5192
5193         my %properties = (
5194             Slice => 'qemu.slice',
5195             KillMode => 'none',
5196             CPUShares => $cpuunits
5197         );
5198
5199         if (my $cpulimit = $conf->{cpulimit}) {
5200             $properties{CPUQuota} = int($cpulimit * 100);
5201         }
5202         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5203
5204         my $run_qemu = sub {
5205             PVE::Tools::run_fork sub {
5206                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5207                 run_command($cmd, %run_params);
5208             };
5209         };
5210
5211         if ($conf->{hugepages}) {
5212
5213             my $code = sub {
5214                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5215                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5216
5217                 PVE::QemuServer::Memory::hugepages_mount();
5218                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5219
5220                 eval { $run_qemu->() };
5221                 if (my $err = $@) {
5222                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5223                     die $err;
5224                 }
5225
5226                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5227             };
5228             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5229
5230         } else {
5231             eval { $run_qemu->() };
5232         }
5233
5234         if (my $err = $@) {
5235             # deactivate volumes if start fails
5236             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5237             die "start failed: $err";
5238         }
5239
5240         print "migration listens on $migrate_uri\n" if $migrate_uri;
5241
5242         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5243             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5244             warn $@ if $@;
5245         }
5246
5247         #start nbd server for storage migration
5248         if ($targetstorage) {
5249             my $nodename = PVE::INotify::nodename();
5250             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5251             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5252             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5253             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5254
5255             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5256
5257             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5258
5259             foreach my $opt (sort keys %$local_volumes) {
5260                 my $volid = $local_volumes->{$opt};
5261                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5262                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5263                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5264             }
5265         }
5266
5267         if ($migratedfrom) {
5268             eval {
5269                 set_migration_caps($vmid);
5270             };
5271             warn $@ if $@;
5272
5273             if ($spice_port) {
5274                 print "spice listens on port $spice_port\n";
5275                 if ($spice_ticket) {
5276                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5277                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5278                 }
5279             }
5280
5281         } else {
5282             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5283                 if !$statefile && $conf->{balloon};
5284
5285             foreach my $opt (keys %$conf) {
5286                 next if $opt !~  m/^net\d+$/;
5287                 my $nicconf = parse_net($conf->{$opt});
5288                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5289             }
5290         }
5291
5292         vm_mon_cmd_nocheck($vmid, 'qom-set',
5293                     path => "machine/peripheral/balloon0",
5294                     property => "guest-stats-polling-interval",
5295                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5296
5297     });
5298 }
5299
5300 sub vm_mon_cmd {
5301     my ($vmid, $execute, %params) = @_;
5302
5303     my $cmd = { execute => $execute, arguments => \%params };
5304     vm_qmp_command($vmid, $cmd);
5305 }
5306
5307 sub vm_mon_cmd_nocheck {
5308     my ($vmid, $execute, %params) = @_;
5309
5310     my $cmd = { execute => $execute, arguments => \%params };
5311     vm_qmp_command($vmid, $cmd, 1);
5312 }
5313
5314 sub vm_qmp_command {
5315     my ($vmid, $cmd, $nocheck) = @_;
5316
5317     my $res;
5318
5319     my $timeout;
5320     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5321         $timeout = $cmd->{arguments}->{timeout};
5322         delete $cmd->{arguments}->{timeout};
5323     }
5324
5325     eval {
5326         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5327         my $sname = qmp_socket($vmid);
5328         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5329             my $qmpclient = PVE::QMPClient->new();
5330
5331             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5332         } else {
5333             die "unable to open monitor socket\n";
5334         }
5335     };
5336     if (my $err = $@) {
5337         syslog("err", "VM $vmid qmp command failed - $err");
5338         die $err;
5339     }
5340
5341     return $res;
5342 }
5343
5344 sub vm_human_monitor_command {
5345     my ($vmid, $cmdline) = @_;
5346
5347     my $res;
5348
5349     my $cmd = {
5350         execute => 'human-monitor-command',
5351         arguments => { 'command-line' => $cmdline},
5352     };
5353
5354     return vm_qmp_command($vmid, $cmd);
5355 }
5356
5357 sub vm_commandline {
5358     my ($storecfg, $vmid) = @_;
5359
5360     my $conf = PVE::QemuConfig->load_config($vmid);
5361
5362     my $defaults = load_defaults();
5363
5364     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5365
5366     return PVE::Tools::cmd2string($cmd);
5367 }
5368
5369 sub vm_reset {
5370     my ($vmid, $skiplock) = @_;
5371
5372     PVE::QemuConfig->lock_config($vmid, sub {
5373
5374         my $conf = PVE::QemuConfig->load_config($vmid);
5375
5376         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5377
5378         vm_mon_cmd($vmid, "system_reset");
5379     });
5380 }
5381
5382 sub get_vm_volumes {
5383     my ($conf) = @_;
5384
5385     my $vollist = [];
5386     foreach_volid($conf, sub {
5387         my ($volid, $attr) = @_;
5388
5389         return if $volid =~ m|^/|;
5390
5391         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5392         return if !$sid;
5393
5394         push @$vollist, $volid;
5395     });
5396
5397     return $vollist;
5398 }
5399
5400 sub vm_stop_cleanup {
5401     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5402
5403     eval {
5404
5405         if (!$keepActive) {
5406             my $vollist = get_vm_volumes($conf);
5407             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5408         }
5409
5410         foreach my $ext (qw(mon qmp pid vnc qga)) {
5411             unlink "/var/run/qemu-server/${vmid}.$ext";
5412         }
5413
5414         foreach my $key (keys %$conf) {
5415             next if $key !~ m/^hostpci(\d+)$/;
5416             my $hostpciindex = $1;
5417             my $d = parse_hostpci($conf->{$key});
5418             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5419
5420             foreach my $pci (@{$d->{pciid}}) {
5421                 my $pciid = $pci->{id};
5422                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5423             }
5424         }
5425
5426         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5427     };
5428     warn $@ if $@; # avoid errors - just warn
5429 }
5430
5431 # Note: use $nockeck to skip tests if VM configuration file exists.
5432 # We need that when migration VMs to other nodes (files already moved)
5433 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5434 sub vm_stop {
5435     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5436
5437     $force = 1 if !defined($force) && !$shutdown;
5438
5439     if ($migratedfrom){
5440         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5441         kill 15, $pid if $pid;
5442         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5443         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5444         return;
5445     }
5446
5447     PVE::QemuConfig->lock_config($vmid, sub {
5448
5449         my $pid = check_running($vmid, $nocheck);
5450         return if !$pid;
5451
5452         my $conf;
5453         if (!$nocheck) {
5454             $conf = PVE::QemuConfig->load_config($vmid);
5455             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5456             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5457                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5458                 $timeout = $opts->{down} if $opts->{down};
5459             }
5460         }
5461
5462         $timeout = 60 if !defined($timeout);
5463
5464         eval {
5465             if ($shutdown) {
5466                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5467                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5468                 } else {
5469                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5470                 }
5471             } else {
5472                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5473             }
5474         };
5475         my $err = $@;
5476
5477         if (!$err) {
5478             my $count = 0;
5479             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5480                 $count++;
5481                 sleep 1;
5482             }
5483
5484             if ($count >= $timeout) {
5485                 if ($force) {
5486                     warn "VM still running - terminating now with SIGTERM\n";
5487                     kill 15, $pid;
5488                 } else {
5489                     die "VM quit/powerdown failed - got timeout\n";
5490                 }
5491             } else {
5492                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5493                 return;
5494             }
5495         } else {
5496             if ($force) {
5497                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5498                 kill 15, $pid;
5499             } else {
5500                 die "VM quit/powerdown failed\n";
5501             }
5502         }
5503
5504         # wait again
5505         $timeout = 10;
5506
5507         my $count = 0;
5508         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5509             $count++;
5510             sleep 1;
5511         }
5512
5513         if ($count >= $timeout) {
5514             warn "VM still running - terminating now with SIGKILL\n";
5515             kill 9, $pid;
5516             sleep 1;
5517         }
5518
5519         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5520    });
5521 }
5522
5523 sub vm_suspend {
5524     my ($vmid, $skiplock) = @_;
5525
5526     PVE::QemuConfig->lock_config($vmid, sub {
5527
5528         my $conf = PVE::QemuConfig->load_config($vmid);
5529
5530         PVE::QemuConfig->check_lock($conf)
5531             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5532
5533         vm_mon_cmd($vmid, "stop");
5534     });
5535 }
5536
5537 sub vm_resume {
5538     my ($vmid, $skiplock, $nocheck) = @_;
5539
5540     PVE::QemuConfig->lock_config($vmid, sub {
5541
5542         my $res = vm_mon_cmd($vmid, 'query-status');
5543         my $resume_cmd = 'cont';
5544
5545         if ($res->{status} && $res->{status} eq 'suspended') {
5546             $resume_cmd = 'system_wakeup';
5547         }
5548
5549         if (!$nocheck) {
5550
5551             my $conf = PVE::QemuConfig->load_config($vmid);
5552
5553             PVE::QemuConfig->check_lock($conf)
5554                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5555
5556             vm_mon_cmd($vmid, $resume_cmd);
5557
5558         } else {
5559             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5560         }
5561     });
5562 }
5563
5564 sub vm_sendkey {
5565     my ($vmid, $skiplock, $key) = @_;
5566
5567     PVE::QemuConfig->lock_config($vmid, sub {
5568
5569         my $conf = PVE::QemuConfig->load_config($vmid);
5570
5571         # there is no qmp command, so we use the human monitor command
5572         vm_human_monitor_command($vmid, "sendkey $key");
5573     });
5574 }
5575
5576 sub vm_destroy {
5577     my ($storecfg, $vmid, $skiplock) = @_;
5578
5579     PVE::QemuConfig->lock_config($vmid, sub {
5580
5581         my $conf = PVE::QemuConfig->load_config($vmid);
5582
5583         if (!check_running($vmid)) {
5584             destroy_vm($storecfg, $vmid, undef, $skiplock);
5585         } else {
5586             die "VM $vmid is running - destroy failed\n";
5587         }
5588     });
5589 }
5590
5591 # vzdump restore implementaion
5592
5593 sub tar_archive_read_firstfile {
5594     my $archive = shift;
5595
5596     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5597
5598     # try to detect archive type first
5599     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5600         die "unable to open file '$archive'\n";
5601     my $firstfile = <$fh>;
5602     kill 15, $pid;
5603     close $fh;
5604
5605     die "ERROR: archive contaions no data\n" if !$firstfile;
5606     chomp $firstfile;
5607
5608     return $firstfile;
5609 }
5610
5611 sub tar_restore_cleanup {
5612     my ($storecfg, $statfile) = @_;
5613
5614     print STDERR "starting cleanup\n";
5615
5616     if (my $fd = IO::File->new($statfile, "r")) {
5617         while (defined(my $line = <$fd>)) {
5618             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5619                 my $volid = $2;
5620                 eval {
5621                     if ($volid =~ m|^/|) {
5622                         unlink $volid || die 'unlink failed\n';
5623                     } else {
5624                         PVE::Storage::vdisk_free($storecfg, $volid);
5625                     }
5626                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5627                 };
5628                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5629             } else {
5630                 print STDERR "unable to parse line in statfile - $line";
5631             }
5632         }
5633         $fd->close();
5634     }
5635 }
5636
5637 sub restore_archive {
5638     my ($archive, $vmid, $user, $opts) = @_;
5639
5640     my $format = $opts->{format};
5641     my $comp;
5642
5643     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5644         $format = 'tar' if !$format;
5645         $comp = 'gzip';
5646     } elsif ($archive =~ m/\.tar$/) {
5647         $format = 'tar' if !$format;
5648     } elsif ($archive =~ m/.tar.lzo$/) {
5649         $format = 'tar' if !$format;
5650         $comp = 'lzop';
5651     } elsif ($archive =~ m/\.vma$/) {
5652         $format = 'vma' if !$format;
5653     } elsif ($archive =~ m/\.vma\.gz$/) {
5654         $format = 'vma' if !$format;
5655         $comp = 'gzip';
5656     } elsif ($archive =~ m/\.vma\.lzo$/) {
5657         $format = 'vma' if !$format;
5658         $comp = 'lzop';
5659     } else {
5660         $format = 'vma' if !$format; # default
5661     }
5662
5663     # try to detect archive format
5664     if ($format eq 'tar') {
5665         return restore_tar_archive($archive, $vmid, $user, $opts);
5666     } else {
5667         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5668     }
5669 }
5670
5671 sub restore_update_config_line {
5672     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5673
5674     return if $line =~ m/^\#qmdump\#/;
5675     return if $line =~ m/^\#vzdump\#/;
5676     return if $line =~ m/^lock:/;
5677     return if $line =~ m/^unused\d+:/;
5678     return if $line =~ m/^parent:/;
5679     return if $line =~ m/^template:/; # restored VM is never a template
5680
5681     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5682     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5683         # try to convert old 1.X settings
5684         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5685         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5686             my ($model, $macaddr) = split(/\=/, $devconfig);
5687             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5688             my $net = {
5689                 model => $model,
5690                 bridge => "vmbr$ind",
5691                 macaddr => $macaddr,
5692             };
5693             my $netstr = print_net($net);
5694
5695             print $outfd "net$cookie->{netcount}: $netstr\n";
5696             $cookie->{netcount}++;
5697         }
5698     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5699         my ($id, $netstr) = ($1, $2);
5700         my $net = parse_net($netstr);
5701         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5702         $netstr = print_net($net);
5703         print $outfd "$id: $netstr\n";
5704     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5705         my $virtdev = $1;
5706         my $value = $3;
5707         my $di = parse_drive($virtdev, $value);
5708         if (defined($di->{backup}) && !$di->{backup}) {
5709             print $outfd "#$line";
5710         } elsif ($map->{$virtdev}) {
5711             delete $di->{format}; # format can change on restore
5712             $di->{file} = $map->{$virtdev};
5713             $value = print_drive($vmid, $di);
5714             print $outfd "$virtdev: $value\n";
5715         } else {
5716             print $outfd $line;
5717         }
5718     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5719         my $vmgenid = $1;
5720         if ($vmgenid ne '0') {
5721             # always generate a new vmgenid if there was a valid one setup
5722             $vmgenid = generate_uuid();
5723         }
5724         print $outfd "vmgenid: $vmgenid\n";
5725     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5726         my ($uuid, $uuid_str);
5727         UUID::generate($uuid);
5728         UUID::unparse($uuid, $uuid_str);
5729         my $smbios1 = parse_smbios1($2);
5730         $smbios1->{uuid} = $uuid_str;
5731         print $outfd $1.print_smbios1($smbios1)."\n";
5732     } else {
5733         print $outfd $line;
5734     }
5735 }
5736
5737 sub scan_volids {
5738     my ($cfg, $vmid) = @_;
5739
5740     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5741
5742     my $volid_hash = {};
5743     foreach my $storeid (keys %$info) {
5744         foreach my $item (@{$info->{$storeid}}) {
5745             next if !($item->{volid} && $item->{size});
5746             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5747             $volid_hash->{$item->{volid}} = $item;
5748         }
5749     }
5750
5751     return $volid_hash;
5752 }
5753
5754 sub is_volume_in_use {
5755     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5756
5757     my $path = PVE::Storage::path($storecfg, $volid);
5758
5759     my $scan_config = sub {
5760         my ($cref, $snapname) = @_;
5761
5762         foreach my $key (keys %$cref) {
5763             my $value = $cref->{$key};
5764             if (is_valid_drivename($key)) {
5765                 next if $skip_drive && $key eq $skip_drive;
5766                 my $drive = parse_drive($key, $value);
5767                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5768                 return 1 if $volid eq $drive->{file};
5769                 if ($drive->{file} =~ m!^/!) {
5770                     return 1 if $drive->{file} eq $path;
5771                 } else {
5772                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5773                     next if !$storeid;
5774                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5775                     next if !$scfg;
5776                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5777                 }
5778             }
5779         }
5780
5781         return 0;
5782     };
5783
5784     return 1 if &$scan_config($conf);
5785
5786     undef $skip_drive;
5787
5788     foreach my $snapname (keys %{$conf->{snapshots}}) {
5789         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5790     }
5791
5792     return 0;
5793 }
5794
5795 sub update_disksize {
5796     my ($vmid, $conf, $volid_hash) = @_;
5797
5798     my $changes;
5799     my $prefix = "VM $vmid:";
5800
5801     # used and unused disks
5802     my $referenced = {};
5803
5804     # Note: it is allowed to define multiple storages with same path (alias), so
5805     # we need to check both 'volid' and real 'path' (two different volid can point
5806     # to the same path).
5807
5808     my $referencedpath = {};
5809
5810     # update size info
5811     foreach my $opt (keys %$conf) {
5812         if (is_valid_drivename($opt)) {
5813             my $drive = parse_drive($opt, $conf->{$opt});
5814             my $volid = $drive->{file};
5815             next if !$volid;
5816
5817             $referenced->{$volid} = 1;
5818             if ($volid_hash->{$volid} &&
5819                 (my $path = $volid_hash->{$volid}->{path})) {
5820                 $referencedpath->{$path} = 1;
5821             }
5822
5823             next if drive_is_cdrom($drive);
5824             next if !$volid_hash->{$volid};
5825
5826             $drive->{size} = $volid_hash->{$volid}->{size};
5827             my $new = print_drive($vmid, $drive);
5828             if ($new ne $conf->{$opt}) {
5829                 $changes = 1;
5830                 $conf->{$opt} = $new;
5831                 print "$prefix update disk '$opt' information.\n";
5832             }
5833         }
5834     }
5835
5836     # remove 'unusedX' entry if volume is used
5837     foreach my $opt (keys %$conf) {
5838         next if $opt !~ m/^unused\d+$/;
5839         my $volid = $conf->{$opt};
5840         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5841         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5842             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5843             $changes = 1;
5844             delete $conf->{$opt};
5845         }
5846
5847         $referenced->{$volid} = 1;
5848         $referencedpath->{$path} = 1 if $path;
5849     }
5850
5851     foreach my $volid (sort keys %$volid_hash) {
5852         next if $volid =~ m/vm-$vmid-state-/;
5853         next if $referenced->{$volid};
5854         my $path = $volid_hash->{$volid}->{path};
5855         next if !$path; # just to be sure
5856         next if $referencedpath->{$path};
5857         $changes = 1;
5858         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5859         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5860         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5861     }
5862
5863     return $changes;
5864 }
5865
5866 sub rescan {
5867     my ($vmid, $nolock, $dryrun) = @_;
5868
5869     my $cfg = PVE::Storage::config();
5870
5871     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5872     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5873     foreach my $stor (keys %{$cfg->{ids}}) {
5874         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5875     }
5876
5877     print "rescan volumes...\n";
5878     my $volid_hash = scan_volids($cfg, $vmid);
5879
5880     my $updatefn =  sub {
5881         my ($vmid) = @_;
5882
5883         my $conf = PVE::QemuConfig->load_config($vmid);
5884
5885         PVE::QemuConfig->check_lock($conf);
5886
5887         my $vm_volids = {};
5888         foreach my $volid (keys %$volid_hash) {
5889             my $info = $volid_hash->{$volid};
5890             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5891         }
5892
5893         my $changes = update_disksize($vmid, $conf, $vm_volids);
5894
5895         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5896     };
5897
5898     if (defined($vmid)) {
5899         if ($nolock) {
5900             &$updatefn($vmid);
5901         } else {
5902             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5903         }
5904     } else {
5905         my $vmlist = config_list();
5906         foreach my $vmid (keys %$vmlist) {
5907             if ($nolock) {
5908                 &$updatefn($vmid);
5909             } else {
5910                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5911             }
5912         }
5913     }
5914 }
5915
5916 sub restore_vma_archive {
5917     my ($archive, $vmid, $user, $opts, $comp) = @_;
5918
5919     my $readfrom = $archive;
5920
5921     my $cfg = PVE::Storage::config();
5922     my $commands = [];
5923     my $bwlimit = $opts->{bwlimit};
5924
5925     my $dbg_cmdstring = '';
5926     my $add_pipe = sub {
5927         my ($cmd) = @_;
5928         push @$commands, $cmd;
5929         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5930         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5931         $readfrom = '-';
5932     };
5933
5934     my $input = undef;
5935     if ($archive eq '-') {
5936         $input = '<&STDIN';
5937     } else {
5938         # If we use a backup from a PVE defined storage we also consider that
5939         # storage's rate limit:
5940         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5941         if (defined($volid)) {
5942             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5943             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5944             if ($readlimit) {
5945                 print STDERR "applying read rate limit: $readlimit\n";
5946                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5947                 $add_pipe->($cstream);
5948             }
5949         }
5950     }
5951
5952     if ($comp) {
5953         my $cmd;
5954         if ($comp eq 'gzip') {
5955             $cmd = ['zcat', $readfrom];
5956         } elsif ($comp eq 'lzop') {
5957             $cmd = ['lzop', '-d', '-c', $readfrom];
5958         } else {
5959             die "unknown compression method '$comp'\n";
5960         }
5961         $add_pipe->($cmd);
5962     }
5963
5964     my $tmpdir = "/var/tmp/vzdumptmp$$";
5965     rmtree $tmpdir;
5966
5967     # disable interrupts (always do cleanups)
5968     local $SIG{INT} =
5969         local $SIG{TERM} =
5970         local $SIG{QUIT} =
5971         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5972
5973     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5974     POSIX::mkfifo($mapfifo, 0600);
5975     my $fifofh;
5976
5977     my $openfifo = sub {
5978         open($fifofh, '>', $mapfifo) || die $!;
5979     };
5980
5981     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5982
5983     my $oldtimeout;
5984     my $timeout = 5;
5985
5986     my $devinfo = {};
5987
5988     my $rpcenv = PVE::RPCEnvironment::get();
5989
5990     my $conffile = PVE::QemuConfig->config_file($vmid);
5991     my $tmpfn = "$conffile.$$.tmp";
5992
5993     # Note: $oldconf is undef if VM does not exists
5994     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5995     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5996
5997     my %storage_limits;
5998
5999     my $print_devmap = sub {
6000         my $virtdev_hash = {};
6001
6002         my $cfgfn = "$tmpdir/qemu-server.conf";
6003
6004         # we can read the config - that is already extracted
6005         my $fh = IO::File->new($cfgfn, "r") ||
6006             "unable to read qemu-server.conf - $!\n";
6007
6008         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6009         if (-f $fwcfgfn) {
6010             my $pve_firewall_dir = '/etc/pve/firewall';
6011             mkdir $pve_firewall_dir; # make sure the dir exists
6012             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6013         }
6014
6015         while (defined(my $line = <$fh>)) {
6016             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6017                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6018                 die "archive does not contain data for drive '$virtdev'\n"
6019                     if !$devinfo->{$devname};
6020                 if (defined($opts->{storage})) {
6021                     $storeid = $opts->{storage} || 'local';
6022                 } elsif (!$storeid) {
6023                     $storeid = 'local';
6024                 }
6025                 $format = 'raw' if !$format;
6026                 $devinfo->{$devname}->{devname} = $devname;
6027                 $devinfo->{$devname}->{virtdev} = $virtdev;
6028                 $devinfo->{$devname}->{format} = $format;
6029                 $devinfo->{$devname}->{storeid} = $storeid;
6030
6031                 # check permission on storage
6032                 my $pool = $opts->{pool}; # todo: do we need that?
6033                 if ($user ne 'root@pam') {
6034                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6035                 }
6036
6037                 $storage_limits{$storeid} = $bwlimit;
6038
6039                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6040             }
6041         }
6042
6043         foreach my $key (keys %storage_limits) {
6044             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6045             next if !$limit;
6046             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6047             $storage_limits{$key} = $limit * 1024;
6048         }
6049
6050         foreach my $devname (keys %$devinfo) {
6051             die "found no device mapping information for device '$devname'\n"
6052                 if !$devinfo->{$devname}->{virtdev};
6053         }
6054
6055         # create empty/temp config
6056         if ($oldconf) {
6057             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6058             foreach_drive($oldconf, sub {
6059                 my ($ds, $drive) = @_;
6060
6061                 return if drive_is_cdrom($drive);
6062
6063                 my $volid = $drive->{file};
6064
6065                 return if !$volid || $volid =~ m|^/|;
6066
6067                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6068                 return if !$path || !$owner || ($owner != $vmid);
6069
6070                 # Note: only delete disk we want to restore
6071                 # other volumes will become unused
6072                 if ($virtdev_hash->{$ds}) {
6073                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6074                     if (my $err = $@) {
6075                         warn $err;
6076                     }
6077                 }
6078             });
6079
6080             # delete vmstate files
6081             # since after the restore we have no snapshots anymore
6082             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6083                 my $snap = $oldconf->{snapshots}->{$snapname};
6084                 if ($snap->{vmstate}) {
6085                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6086                     if (my $err = $@) {
6087                         warn $err;
6088                     }
6089                 }
6090             }
6091         }
6092
6093         my $map = {};
6094         foreach my $virtdev (sort keys %$virtdev_hash) {
6095             my $d = $virtdev_hash->{$virtdev};
6096             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6097             my $storeid = $d->{storeid};
6098             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6099
6100             my $map_opts = '';
6101             if (my $limit = $storage_limits{$storeid}) {
6102                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6103             }
6104
6105             # test if requested format is supported
6106             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6107             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6108             $d->{format} = $defFormat if !$supported;
6109
6110             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6111                                                   $d->{format}, undef, $alloc_size);
6112             print STDERR "new volume ID is '$volid'\n";
6113             $d->{volid} = $volid;
6114             my $path = PVE::Storage::path($cfg, $volid);
6115
6116             PVE::Storage::activate_volumes($cfg,[$volid]);
6117
6118             my $write_zeros = 1;
6119             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6120                 $write_zeros = 0;
6121             }
6122
6123             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6124
6125             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6126             $map->{$virtdev} = $volid;
6127         }
6128
6129         $fh->seek(0, 0) || die "seek failed - $!\n";
6130
6131         my $outfd = new IO::File ($tmpfn, "w") ||
6132             die "unable to write config for VM $vmid\n";
6133
6134         my $cookie = { netcount => 0 };
6135         while (defined(my $line = <$fh>)) {
6136             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6137         }
6138
6139         $fh->close();
6140         $outfd->close();
6141     };
6142
6143     eval {
6144         # enable interrupts
6145         local $SIG{INT} =
6146             local $SIG{TERM} =
6147             local $SIG{QUIT} =
6148             local $SIG{HUP} =
6149             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6150         local $SIG{ALRM} = sub { die "got timeout\n"; };
6151
6152         $oldtimeout = alarm($timeout);
6153
6154         my $parser = sub {
6155             my $line = shift;
6156
6157             print "$line\n";
6158
6159             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6160                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6161                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6162             } elsif ($line =~ m/^CTIME: /) {
6163                 # we correctly received the vma config, so we can disable
6164                 # the timeout now for disk allocation (set to 10 minutes, so
6165                 # that we always timeout if something goes wrong)
6166                 alarm(600);
6167                 &$print_devmap();
6168                 print $fifofh "done\n";
6169                 my $tmp = $oldtimeout || 0;
6170                 $oldtimeout = undef;
6171                 alarm($tmp);
6172                 close($fifofh);
6173             }
6174         };
6175
6176         print "restore vma archive: $dbg_cmdstring\n";
6177         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6178     };
6179     my $err = $@;
6180
6181     alarm($oldtimeout) if $oldtimeout;
6182
6183     my $vollist = [];
6184     foreach my $devname (keys %$devinfo) {
6185         my $volid = $devinfo->{$devname}->{volid};
6186         push @$vollist, $volid if $volid;
6187     }
6188
6189     PVE::Storage::deactivate_volumes($cfg, $vollist);
6190
6191     unlink $mapfifo;
6192
6193     if ($err) {
6194         rmtree $tmpdir;
6195         unlink $tmpfn;
6196
6197         foreach my $devname (keys %$devinfo) {
6198             my $volid = $devinfo->{$devname}->{volid};
6199             next if !$volid;
6200             eval {
6201                 if ($volid =~ m|^/|) {
6202                     unlink $volid || die 'unlink failed\n';
6203                 } else {
6204                     PVE::Storage::vdisk_free($cfg, $volid);
6205                 }
6206                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6207             };
6208             print STDERR "unable to cleanup '$volid' - $@" if $@;
6209         }
6210         die $err;
6211     }
6212
6213     rmtree $tmpdir;
6214
6215     rename($tmpfn, $conffile) ||
6216         die "unable to commit configuration file '$conffile'\n";
6217
6218     PVE::Cluster::cfs_update(); # make sure we read new file
6219
6220     eval { rescan($vmid, 1); };
6221     warn $@ if $@;
6222 }
6223
6224 sub restore_tar_archive {
6225     my ($archive, $vmid, $user, $opts) = @_;
6226
6227     if ($archive ne '-') {
6228         my $firstfile = tar_archive_read_firstfile($archive);
6229         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6230             if $firstfile ne 'qemu-server.conf';
6231     }
6232
6233     my $storecfg = PVE::Storage::config();
6234
6235     # destroy existing data - keep empty config
6236     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6237     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6238
6239     my $tocmd = "/usr/lib/qemu-server/qmextract";
6240
6241     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6242     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6243     $tocmd .= ' --prealloc' if $opts->{prealloc};
6244     $tocmd .= ' --info' if $opts->{info};
6245
6246     # tar option "xf" does not autodetect compression when read from STDIN,
6247     # so we pipe to zcat
6248     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6249         PVE::Tools::shellquote("--to-command=$tocmd");
6250
6251     my $tmpdir = "/var/tmp/vzdumptmp$$";
6252     mkpath $tmpdir;
6253
6254     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6255     local $ENV{VZDUMP_VMID} = $vmid;
6256     local $ENV{VZDUMP_USER} = $user;
6257
6258     my $conffile = PVE::QemuConfig->config_file($vmid);
6259     my $tmpfn = "$conffile.$$.tmp";
6260
6261     # disable interrupts (always do cleanups)
6262     local $SIG{INT} =
6263         local $SIG{TERM} =
6264         local $SIG{QUIT} =
6265         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6266
6267     eval {
6268         # enable interrupts
6269         local $SIG{INT} =
6270             local $SIG{TERM} =
6271             local $SIG{QUIT} =
6272             local $SIG{HUP} =
6273             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6274
6275         if ($archive eq '-') {
6276             print "extracting archive from STDIN\n";
6277             run_command($cmd, input => "<&STDIN");
6278         } else {
6279             print "extracting archive '$archive'\n";
6280             run_command($cmd);
6281         }
6282
6283         return if $opts->{info};
6284
6285         # read new mapping
6286         my $map = {};
6287         my $statfile = "$tmpdir/qmrestore.stat";
6288         if (my $fd = IO::File->new($statfile, "r")) {
6289             while (defined (my $line = <$fd>)) {
6290                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6291                     $map->{$1} = $2 if $1;
6292                 } else {
6293                     print STDERR "unable to parse line in statfile - $line\n";
6294                 }
6295             }
6296             $fd->close();
6297         }
6298
6299         my $confsrc = "$tmpdir/qemu-server.conf";
6300
6301         my $srcfd = new IO::File($confsrc, "r") ||
6302             die "unable to open file '$confsrc'\n";
6303
6304         my $outfd = new IO::File ($tmpfn, "w") ||
6305             die "unable to write config for VM $vmid\n";
6306
6307         my $cookie = { netcount => 0 };
6308         while (defined (my $line = <$srcfd>)) {
6309             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6310         }
6311
6312         $srcfd->close();
6313         $outfd->close();
6314     };
6315     my $err = $@;
6316
6317     if ($err) {
6318
6319         unlink $tmpfn;
6320
6321         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6322
6323         die $err;
6324     }
6325
6326     rmtree $tmpdir;
6327
6328     rename $tmpfn, $conffile ||
6329         die "unable to commit configuration file '$conffile'\n";
6330
6331     PVE::Cluster::cfs_update(); # make sure we read new file
6332
6333     eval { rescan($vmid, 1); };
6334     warn $@ if $@;
6335 };
6336
6337 sub foreach_storage_used_by_vm {
6338     my ($conf, $func) = @_;
6339
6340     my $sidhash = {};
6341
6342     foreach_drive($conf, sub {
6343         my ($ds, $drive) = @_;
6344         return if drive_is_cdrom($drive);
6345
6346         my $volid = $drive->{file};
6347
6348         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6349         $sidhash->{$sid} = $sid if $sid;
6350     });
6351
6352     foreach my $sid (sort keys %$sidhash) {
6353         &$func($sid);
6354     }
6355 }
6356
6357 sub do_snapshots_with_qemu {
6358     my ($storecfg, $volid) = @_;
6359
6360     my $storage_name = PVE::Storage::parse_volume_id($volid);
6361
6362     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6363         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6364         return 1;
6365     }
6366
6367     if ($volid =~ m/\.(qcow2|qed)$/){
6368         return 1;
6369     }
6370
6371     return undef;
6372 }
6373
6374 sub qga_check_running {
6375     my ($vmid, $nowarn) = @_;
6376
6377     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6378     if ($@) {
6379         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6380         return 0;
6381     }
6382     return 1;
6383 }
6384
6385 sub template_create {
6386     my ($vmid, $conf, $disk) = @_;
6387
6388     my $storecfg = PVE::Storage::config();
6389
6390     foreach_drive($conf, sub {
6391         my ($ds, $drive) = @_;
6392
6393         return if drive_is_cdrom($drive);
6394         return if $disk && $ds ne $disk;
6395
6396         my $volid = $drive->{file};
6397         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6398
6399         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6400         $drive->{file} = $voliddst;
6401         $conf->{$ds} = print_drive($vmid, $drive);
6402         PVE::QemuConfig->write_config($vmid, $conf);
6403     });
6404 }
6405
6406 sub qemu_img_convert {
6407     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6408
6409     my $storecfg = PVE::Storage::config();
6410     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6411     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6412
6413     if ($src_storeid && $dst_storeid) {
6414
6415         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6416
6417         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6418         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6419
6420         my $src_format = qemu_img_format($src_scfg, $src_volname);
6421         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6422
6423         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6424         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6425
6426         my $cmd = [];
6427         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6428         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6429         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6430         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6431         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6432         if ($is_zero_initialized) {
6433             push @$cmd, "zeroinit:$dst_path";
6434         } else {
6435             push @$cmd, $dst_path;
6436         }
6437
6438         my $parser = sub {
6439             my $line = shift;
6440             if($line =~ m/\((\S+)\/100\%\)/){
6441                 my $percent = $1;
6442                 my $transferred = int($size * $percent / 100);
6443                 my $remaining = $size - $transferred;
6444
6445                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6446             }
6447
6448         };
6449
6450         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6451         my $err = $@;
6452         die "copy failed: $err" if $err;
6453     }
6454 }
6455
6456 sub qemu_img_format {
6457     my ($scfg, $volname) = @_;
6458
6459     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6460         return $1;
6461     } else {
6462         return "raw";
6463     }
6464 }
6465
6466 sub qemu_drive_mirror {
6467     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6468
6469     $jobs = {} if !$jobs;
6470
6471     my $qemu_target;
6472     my $format;
6473     $jobs->{"drive-$drive"} = {};
6474
6475     if ($dst_volid =~ /^nbd:/) {
6476         $qemu_target = $dst_volid;
6477         $format = "nbd";
6478     } else {
6479         my $storecfg = PVE::Storage::config();
6480         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6481
6482         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6483
6484         $format = qemu_img_format($dst_scfg, $dst_volname);
6485
6486         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6487
6488         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6489     }
6490
6491     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6492     $opts->{format} = $format if $format;
6493
6494     print "drive mirror is starting for drive-$drive\n";
6495
6496     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6497
6498     if (my $err = $@) {
6499         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6500         die "mirroring error: $err";
6501     }
6502
6503     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6504 }
6505
6506 sub qemu_drive_mirror_monitor {
6507     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6508
6509     eval {
6510         my $err_complete = 0;
6511
6512         while (1) {
6513             die "storage migration timed out\n" if $err_complete > 300;
6514
6515             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6516
6517             my $running_mirror_jobs = {};
6518             foreach my $stat (@$stats) {
6519                 next if $stat->{type} ne 'mirror';
6520                 $running_mirror_jobs->{$stat->{device}} = $stat;
6521             }
6522
6523             my $readycounter = 0;
6524
6525             foreach my $job (keys %$jobs) {
6526
6527                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6528                     print "$job : finished\n";
6529                     delete $jobs->{$job};
6530                     next;
6531                 }
6532
6533                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6534
6535                 my $busy = $running_mirror_jobs->{$job}->{busy};
6536                 my $ready = $running_mirror_jobs->{$job}->{ready};
6537                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6538                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6539                     my $remaining = $total - $transferred;
6540                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6541
6542                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6543                 }
6544
6545                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6546             }
6547
6548             last if scalar(keys %$jobs) == 0;
6549
6550             if ($readycounter == scalar(keys %$jobs)) {
6551                 print "all mirroring jobs are ready \n";
6552                 last if $skipcomplete; #do the complete later
6553
6554                 if ($vmiddst && $vmiddst != $vmid) {
6555                     my $agent_running = $qga && qga_check_running($vmid);
6556                     if ($agent_running) {
6557                         print "freeze filesystem\n";
6558                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6559                     } else {
6560                         print "suspend vm\n";
6561                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6562                     }
6563
6564                     # if we clone a disk for a new target vm, we don't switch the disk
6565                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6566
6567                     if ($agent_running) {
6568                         print "unfreeze filesystem\n";
6569                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6570                     } else {
6571                         print "resume vm\n";
6572                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6573                     }
6574
6575                     last;
6576                 } else {
6577
6578                     foreach my $job (keys %$jobs) {
6579                         # try to switch the disk if source and destination are on the same guest
6580                         print "$job: Completing block job...\n";
6581
6582                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6583                         if ($@ =~ m/cannot be completed/) {
6584                             print "$job: Block job cannot be completed, try again.\n";
6585                             $err_complete++;
6586                         }else {
6587                             print "$job: Completed successfully.\n";
6588                             $jobs->{$job}->{complete} = 1;
6589                         }
6590                     }
6591                 }
6592             }
6593             sleep 1;
6594         }
6595     };
6596     my $err = $@;
6597
6598     if ($err) {
6599         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6600         die "mirroring error: $err";
6601     }
6602
6603 }
6604
6605 sub qemu_blockjobs_cancel {
6606     my ($vmid, $jobs) = @_;
6607
6608     foreach my $job (keys %$jobs) {
6609         print "$job: Cancelling block job\n";
6610         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6611         $jobs->{$job}->{cancel} = 1;
6612     }
6613
6614     while (1) {
6615         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6616
6617         my $running_jobs = {};
6618         foreach my $stat (@$stats) {
6619             $running_jobs->{$stat->{device}} = $stat;
6620         }
6621
6622         foreach my $job (keys %$jobs) {
6623
6624             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6625                 print "$job: Done.\n";
6626                 delete $jobs->{$job};
6627             }
6628         }
6629
6630         last if scalar(keys %$jobs) == 0;
6631
6632         sleep 1;
6633     }
6634 }
6635
6636 sub clone_disk {
6637     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6638         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6639
6640     my $newvolid;
6641
6642     if (!$full) {
6643         print "create linked clone of drive $drivename ($drive->{file})\n";
6644         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6645         push @$newvollist, $newvolid;
6646     } else {
6647
6648         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6649         $storeid = $storage if $storage;
6650
6651         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6652         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6653
6654         print "create full clone of drive $drivename ($drive->{file})\n";
6655         my $name = undef;
6656         if (drive_is_cloudinit($drive)) {
6657             $name = "vm-$newvmid-cloudinit";
6658             $snapname = undef;
6659             # cloudinit only supports raw and qcow2 atm:
6660             if ($dst_format eq 'qcow2') {
6661                 $name .= '.qcow2';
6662             } elsif ($dst_format ne 'raw') {
6663                 die "clone: unhandled format for cloudinit image\n";
6664             }
6665         }
6666         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6667         push @$newvollist, $newvolid;
6668
6669         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6670
6671         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6672         if (!$running || $snapname) {
6673             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6674         } else {
6675
6676             my $kvmver = get_running_qemu_version ($vmid);
6677             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6678                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6679                     if $drive->{iothread};
6680             }
6681
6682             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6683         }
6684     }
6685
6686     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6687
6688     my $disk = $drive;
6689     $disk->{format} = undef;
6690     $disk->{file} = $newvolid;
6691     $disk->{size} = $size;
6692
6693     return $disk;
6694 }
6695
6696 # this only works if VM is running
6697 sub get_current_qemu_machine {
6698     my ($vmid) = @_;
6699
6700     my $cmd = { execute => 'query-machines', arguments => {} };
6701     my $res = vm_qmp_command($vmid, $cmd);
6702
6703     my ($current, $default);
6704     foreach my $e (@$res) {
6705         $default = $e->{name} if $e->{'is-default'};
6706         $current = $e->{name} if $e->{'is-current'};
6707     }
6708
6709     # fallback to the default machine if current is not supported by qemu
6710     return $current || $default || 'pc';
6711 }
6712
6713 sub get_running_qemu_version {
6714     my ($vmid) = @_;
6715     my $cmd = { execute => 'query-version', arguments => {} };
6716     my $res = vm_qmp_command($vmid, $cmd);
6717     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6718 }
6719
6720 sub qemu_machine_feature_enabled {
6721     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6722
6723     my $current_major;
6724     my $current_minor;
6725
6726     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6727
6728         $current_major = $3;
6729         $current_minor = $4;
6730
6731     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6732
6733         $current_major = $1;
6734         $current_minor = $2;
6735     }
6736
6737     return 1 if $current_major > $version_major ||
6738                 ($current_major == $version_major &&
6739                  $current_minor >= $version_minor);
6740 }
6741
6742 sub qemu_machine_pxe {
6743     my ($vmid, $conf, $machine) = @_;
6744
6745     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6746
6747     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6748         $machine .= '.pxe';
6749     }
6750
6751     return $machine;
6752 }
6753
6754 sub qemu_use_old_bios_files {
6755     my ($machine_type) = @_;
6756
6757     return if !$machine_type;
6758
6759     my $use_old_bios_files = undef;
6760
6761     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6762         $machine_type = $1;
6763         $use_old_bios_files = 1;
6764     } else {
6765         my $kvmver = kvm_user_version();
6766         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6767         # load new efi bios files on migration. So this hack is required to allow
6768         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6769         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6770         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6771     }
6772
6773     return ($use_old_bios_files, $machine_type);
6774 }
6775
6776 sub create_efidisk($$$$$) {
6777     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6778
6779     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6780     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6781
6782     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6783     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6784     PVE::Storage::activate_volumes($storecfg, [$volid]);
6785
6786     my $path = PVE::Storage::path($storecfg, $volid);
6787     eval {
6788         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6789     };
6790     die "Copying EFI vars image failed: $@" if $@;
6791
6792     return ($volid, $vars_size);
6793 }
6794
6795 sub vm_iothreads_list {
6796     my ($vmid) = @_;
6797
6798     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6799
6800     my $iothreads = {};
6801     foreach my $iothread (@$res) {
6802         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6803     }
6804
6805     return $iothreads;
6806 }
6807
6808 sub scsihw_infos {
6809     my ($conf, $drive) = @_;
6810
6811     my $maxdev = 0;
6812
6813     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6814         $maxdev = 7;
6815     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6816         $maxdev = 1;
6817     } else {
6818         $maxdev = 256;
6819     }
6820
6821     my $controller = int($drive->{index} / $maxdev);
6822     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6823
6824     return ($maxdev, $controller, $controller_prefix);
6825 }
6826
6827 sub add_hyperv_enlightenments {
6828     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6829
6830     return if $winversion < 6;
6831     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6832
6833     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6834
6835     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6836         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6837         push @$cpuFlags , 'hv_vapic';
6838         push @$cpuFlags , 'hv_time';
6839     } else {
6840         push @$cpuFlags , 'hv_spinlocks=0xffff';
6841     }
6842
6843     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6844         push @$cpuFlags , 'hv_reset';
6845         push @$cpuFlags , 'hv_vpindex';
6846         push @$cpuFlags , 'hv_runtime';
6847     }
6848
6849     if ($winversion >= 7) {
6850         push @$cpuFlags , 'hv_relaxed';
6851
6852         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6853             push @$cpuFlags , 'hv_synic';
6854             push @$cpuFlags , 'hv_stimer';
6855         }
6856     }
6857 }
6858
6859 sub windows_version {
6860     my ($ostype) = @_;
6861
6862     return 0 if !$ostype;
6863
6864     my $winversion = 0;
6865
6866     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6867         $winversion = 5;
6868     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6869         $winversion = 6;
6870     } elsif ($ostype =~ m/^win(\d+)$/) {
6871         $winversion = $1;
6872     }
6873
6874     return $winversion;
6875 }
6876
6877 sub resolve_dst_disk_format {
6878         my ($storecfg, $storeid, $src_volname, $format) = @_;
6879         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6880
6881         if (!$format) {
6882             # if no target format is specified, use the source disk format as hint
6883             if ($src_volname) {
6884                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6885                 $format = qemu_img_format($scfg, $src_volname);
6886             } else {
6887                 return $defFormat;
6888             }
6889         }
6890
6891         # test if requested format is supported - else use default
6892         my $supported = grep { $_ eq $format } @$validFormats;
6893         $format = $defFormat if !$supported;
6894         return $format;
6895 }
6896
6897 sub resolve_first_disk {
6898     my $conf = shift;
6899     my @disks = PVE::QemuServer::valid_drive_names();
6900     my $firstdisk;
6901     foreach my $ds (reverse @disks) {
6902         next if !$conf->{$ds};
6903         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6904         next if PVE::QemuServer::drive_is_cdrom($disk);
6905         $firstdisk = $ds;
6906     }
6907     return $firstdisk;
6908 }
6909
6910 sub generate_uuid {
6911     my ($uuid, $uuid_str);
6912     UUID::generate($uuid);
6913     UUID::unparse($uuid, $uuid_str);
6914     return $uuid_str;
6915 }
6916
6917 sub generate_smbios1_uuid {
6918     return "uuid=".generate_uuid();
6919 }
6920
6921 sub nbd_stop {
6922     my ($vmid) = @_;
6923
6924     vm_mon_cmd($vmid, 'nbd-server-stop');
6925 }
6926
6927 # bash completion helper
6928
6929 sub complete_backup_archives {
6930     my ($cmdname, $pname, $cvalue) = @_;
6931
6932     my $cfg = PVE::Storage::config();
6933
6934     my $storeid;
6935
6936     if ($cvalue =~ m/^([^:]+):/) {
6937         $storeid = $1;
6938     }
6939
6940     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6941
6942     my $res = [];
6943     foreach my $id (keys %$data) {
6944         foreach my $item (@{$data->{$id}}) {
6945             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6946             push @$res, $item->{volid} if defined($item->{volid});
6947         }
6948     }
6949
6950     return $res;
6951 }
6952
6953 my $complete_vmid_full = sub {
6954     my ($running) = @_;
6955
6956     my $idlist = vmstatus();
6957
6958     my $res = [];
6959
6960     foreach my $id (keys %$idlist) {
6961         my $d = $idlist->{$id};
6962         if (defined($running)) {
6963             next if $d->{template};
6964             next if $running && $d->{status} ne 'running';
6965             next if !$running && $d->{status} eq 'running';
6966         }
6967         push @$res, $id;
6968
6969     }
6970     return $res;
6971 };
6972
6973 sub complete_vmid {
6974     return &$complete_vmid_full();
6975 }
6976
6977 sub complete_vmid_stopped {
6978     return &$complete_vmid_full(0);
6979 }
6980
6981 sub complete_vmid_running {
6982     return &$complete_vmid_full(1);
6983 }
6984
6985 sub complete_storage {
6986
6987     my $cfg = PVE::Storage::config();
6988     my $ids = $cfg->{ids};
6989
6990     my $res = [];
6991     foreach my $sid (keys %$ids) {
6992         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6993         next if !$ids->{$sid}->{content}->{images};
6994         push @$res, $sid;
6995     }
6996
6997     return $res;
6998 }
6999
7000 1;