PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use Time::HiRes qw(gettimeofday);
  40 use File::Copy qw(copy);
  41 use URI::Escape;
  42
  43 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  44 my $OVMF = {
  45     x86_64 => [
  46         "$EDK2_FW_BASE/OVMF_CODE.fd",
  47         "$EDK2_FW_BASE/OVMF_VARS.fd"
  48     ],
  49     aarch64 => [
  50         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  51         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  52     ],
  53 };
  54
  55 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  56
  57 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  58
  59 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  60
  61 # Note about locking: we use flock on the config file protect
  62 # against concurent actions.
  63 # Aditionaly, we have a 'lock' setting in the config file. This
  64 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  65 # allowed when such lock is set. But you can ignore this kind of
  66 # lock with the --skiplock flag.
  67
  68 cfs_register_file('/qemu-server/',
  69                   \&parse_vm_config,
  70                   \&write_vm_config);
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  73     description => "Some command save/restore state from this location.",
  74     type => 'string',
  75     maxLength => 128,
  76     optional => 1,
  77 });
  78
  79 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  80     description => "The name of the snapshot.",
  81     type => 'string', format => 'pve-configid',
  82     maxLength => 40,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $cpu_vendor_list = {
 123     # Intel CPUs
 124     486 => 'GenuineIntel',
 125     pentium => 'GenuineIntel',
 126     pentium2  => 'GenuineIntel',
 127     pentium3  => 'GenuineIntel',
 128     coreduo => 'GenuineIntel',
 129     core2duo => 'GenuineIntel',
 130     Conroe  => 'GenuineIntel',
 131     Penryn  => 'GenuineIntel',
 132     Nehalem  => 'GenuineIntel',
 133     'Nehalem-IBRS'  => 'GenuineIntel',
 134     Westmere => 'GenuineIntel',
 135     'Westmere-IBRS' => 'GenuineIntel',
 136     SandyBridge => 'GenuineIntel',
 137     'SandyBridge-IBRS' => 'GenuineIntel',
 138     IvyBridge => 'GenuineIntel',
 139     'IvyBridge-IBRS' => 'GenuineIntel',
 140     Haswell => 'GenuineIntel',
 141     'Haswell-IBRS' => 'GenuineIntel',
 142     'Haswell-noTSX' => 'GenuineIntel',
 143     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 144     Broadwell => 'GenuineIntel',
 145     'Broadwell-IBRS' => 'GenuineIntel',
 146     'Broadwell-noTSX' => 'GenuineIntel',
 147     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 148     'Skylake-Client' => 'GenuineIntel',
 149     'Skylake-Client-IBRS' => 'GenuineIntel',
 150     'Skylake-Server' => 'GenuineIntel',
 151     'Skylake-Server-IBRS' => 'GenuineIntel',
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 174
 175 my $cpu_fmt = {
 176     cputype => {
 177         description => "Emulated CPU type.",
 178         type => 'string',
 179         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 180         default => 'kvm64',
 181         default_key => 1,
 182     },
 183     hidden => {
 184         description => "Do not identify as a KVM virtual machine.",
 185         type => 'boolean',
 186         optional => 1,
 187         default => 0
 188     },
 189     flags => {
 190         description => "List of additional CPU flags separated by ';'."
 191                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 192                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 193         format_description => '+FLAG[;-FLAG...]',
 194         type => 'string',
 195         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 196         optional => 1,
 197     },
 198 };
 199
 200 my $watchdog_fmt = {
 201     model => {
 202         default_key => 1,
 203         type => 'string',
 204         enum => [qw(i6300esb ib700)],
 205         description => "Watchdog type to emulate.",
 206         default => 'i6300esb',
 207         optional => 1,
 208     },
 209     action => {
 210         type => 'string',
 211         enum => [qw(reset shutdown poweroff pause debug none)],
 212         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 213         optional => 1,
 214     },
 215 };
 216 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 217
 218 my $agent_fmt = {
 219     enabled => {
 220         description => "Enable/disable Qemu GuestAgent.",
 221         type => 'boolean',
 222         default => 0,
 223         default_key => 1,
 224     },
 225     fstrim_cloned_disks => {
 226         description => "Run fstrim after cloning/moving a disk.",
 227         type => 'boolean',
 228         optional => 1,
 229         default => 0
 230     },
 231 };
 232
 233 my $vga_fmt = {
 234     type => {
 235         description => "Select the VGA type.",
 236         type => 'string',
 237         default => 'std',
 238         optional => 1,
 239         default_key => 1,
 240         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 241     },
 242     memory => {
 243         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 244         type => 'integer',
 245         optional => 1,
 246         minimum => 4,
 247         maximum => 512,
 248     },
 249 };
 250
 251 my $confdesc = {
 252     onboot => {
 253         optional => 1,
 254         type => 'boolean',
 255         description => "Specifies whether a VM will be started during system bootup.",
 256         default => 0,
 257     },
 258     autostart => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Automatic restart after crash (currently ignored).",
 262         default => 0,
 263     },
 264     hotplug => {
 265         optional => 1,
 266         type => 'string', format => 'pve-hotplug-features',
 267         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 268         default => 'network,disk,usb',
 269     },
 270     reboot => {
 271         optional => 1,
 272         type => 'boolean',
 273         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 274         default => 1,
 275     },
 276     lock => {
 277         optional => 1,
 278         type => 'string',
 279         description => "Lock/unlock the VM.",
 280         enum => [qw(migrate backup snapshot rollback)],
 281     },
 282     cpulimit => {
 283         optional => 1,
 284         type => 'number',
 285         description => "Limit of CPU usage.",
 286         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 287         minimum => 0,
 288         maximum => 128,
 289         default => 0,
 290     },
 291     cpuunits => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "CPU weight for a VM.",
 295         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 296         minimum => 2,
 297         maximum => 262144,
 298         default => 1024,
 299     },
 300     memory => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 304         minimum => 16,
 305         default => 512,
 306     },
 307     balloon => {
 308         optional => 1,
 309         type => 'integer',
 310         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 311         minimum => 0,
 312     },
 313     shares => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 317         minimum => 0,
 318         maximum => 50000,
 319         default => 1000,
 320     },
 321     keyboard => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 325                        "It should not be necessary to set it.",
 326         enum => PVE::Tools::kvmkeymaplist(),
 327         default => undef,
 328     },
 329     name => {
 330         optional => 1,
 331         type => 'string', format => 'dns-name',
 332         description => "Set a name for the VM. Only used on the configuration web interface.",
 333     },
 334     scsihw => {
 335         optional => 1,
 336         type => 'string',
 337         description => "SCSI controller model",
 338         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 339         default => 'lsi',
 340     },
 341     description => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 345     },
 346     ostype => {
 347         optional => 1,
 348         type => 'string',
 349         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 350         description => "Specify guest operating system.",
 351         verbose_description => <<EODESC,
 352 Specify guest operating system. This is used to enable special
 353 optimization/features for specific operating systems:
 354
 355 [horizontal]
 356 other;; unspecified OS
 357 wxp;; Microsoft Windows XP
 358 w2k;; Microsoft Windows 2000
 359 w2k3;; Microsoft Windows 2003
 360 w2k8;; Microsoft Windows 2008
 361 wvista;; Microsoft Windows Vista
 362 win7;; Microsoft Windows 7
 363 win8;; Microsoft Windows 8/2012/2012r2
 364 win10;; Microsoft Windows 10/2016
 365 l24;; Linux 2.4 Kernel
 366 l26;; Linux 2.6/3.X Kernel
 367 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 368 EODESC
 369     },
 370     boot => {
 371         optional => 1,
 372         type => 'string',
 373         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 374         pattern => '[acdn]{1,4}',
 375         default => 'cdn',
 376     },
 377     bootdisk => {
 378         optional => 1,
 379         type => 'string', format => 'pve-qm-bootdisk',
 380         description => "Enable booting from specified disk.",
 381         pattern => '(ide|sata|scsi|virtio)\d+',
 382     },
 383     smp => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPUs. Please use option -sockets instead.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     sockets => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of CPU sockets.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     cores => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "The number of cores per socket.",
 401         minimum => 1,
 402         default => 1,
 403     },
 404     numa => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Enable/disable NUMA.",
 408         default => 0,
 409     },
 410     hugepages => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Enable/disable hugepages memory.",
 414         enum => [qw(any 2 1024)],
 415     },
 416     vcpus => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "Number of hotplugged vcpus.",
 420         minimum => 1,
 421         default => 0,
 422     },
 423     acpi => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable ACPI.",
 427         default => 1,
 428     },
 429     agent => {
 430         optional => 1,
 431         description => "Enable/disable Qemu GuestAgent and its properties.",
 432         type => 'string',
 433         format => $agent_fmt,
 434     },
 435     kvm => {
 436         optional => 1,
 437         type => 'boolean',
 438         description => "Enable/disable KVM hardware virtualization.",
 439         default => 1,
 440     },
 441     tdf => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable time drift fix.",
 445         default => 0,
 446     },
 447     localtime => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 451     },
 452     freeze => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 456     },
 457     vga => {
 458         optional => 1,
 459         type => 'string', format => $vga_fmt,
 460         description => "Configure the VGA hardware.",
 461         verbose_description => "Configure the VGA Hardware. If you want to use ".
 462             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 463             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 464             "is 'std' for all OS types besides some Windows versions (XP and " .
 465             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 466             "display server. For win* OS you can select how many independent " .
 467             "displays you want, Linux guests can add displays them self.\n".
 468             "You can also run without any graphic card, using a serial device as terminal.",
 469     },
 470     watchdog => {
 471         optional => 1,
 472         type => 'string', format => 'pve-qm-watchdog',
 473         description => "Create a virtual hardware watchdog device.",
 474         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 475             " (by a guest action), the watchdog must be periodically polled " .
 476             "by an agent inside the guest or else the watchdog will reset " .
 477             "the guest (or execute the respective action specified)",
 478     },
 479     startdate => {
 480         optional => 1,
 481         type => 'string',
 482         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 483         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 484         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 485         default => 'now',
 486     },
 487     startup =>  get_standard_option('pve-startup-order'),
 488     template => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable Template.",
 492         default => 0,
 493     },
 494     args => {
 495         optional => 1,
 496         type => 'string',
 497         description => "Arbitrary arguments passed to kvm.",
 498         verbose_description => <<EODESCR,
 499 Arbitrary arguments passed to kvm, for example:
 500
 501 args: -no-reboot -no-hpet
 502
 503 NOTE: this option is for experts only.
 504 EODESCR
 505     },
 506     tablet => {
 507         optional => 1,
 508         type => 'boolean',
 509         default => 1,
 510         description => "Enable/disable the USB tablet device.",
 511         verbose_description => "Enable/disable the USB tablet device. This device is " .
 512             "usually needed to allow absolute mouse positioning with VNC. " .
 513             "Else the mouse runs out of sync with normal VNC clients. " .
 514             "If you're running lots of console-only guests on one host, " .
 515             "you may consider disabling this to save some context switches. " .
 516             "This is turned off by default if you use spice (-vga=qxl).",
 517     },
 518     migrate_speed => {
 519         optional => 1,
 520         type => 'integer',
 521         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 522         minimum => 0,
 523         default => 0,
 524     },
 525     migrate_downtime => {
 526         optional => 1,
 527         type => 'number',
 528         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 529         minimum => 0,
 530         default => 0.1,
 531     },
 532     cdrom => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-ide',
 535         typetext => '<volume>',
 536         description => "This is an alias for option -ide2",
 537     },
 538     cpu => {
 539         optional => 1,
 540         description => "Emulated CPU type.",
 541         type => 'string',
 542         format => $cpu_fmt,
 543     },
 544     parent => get_standard_option('pve-snapshot-name', {
 545         optional => 1,
 546         description => "Parent snapshot name. This is used internally, and should not be modified.",
 547     }),
 548     snaptime => {
 549         optional => 1,
 550         description => "Timestamp for snapshots.",
 551         type => 'integer',
 552         minimum => 0,
 553     },
 554     vmstate => {
 555         optional => 1,
 556         type => 'string', format => 'pve-volume-id',
 557         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 558     },
 559     vmstatestorage => get_standard_option('pve-storage-id', {
 560         description => "Default storage for VM state volumes/files.",
 561         optional => 1,
 562     }),
 563     runningmachine => get_standard_option('pve-qemu-machine', {
 564         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 565     }),
 566     machine => get_standard_option('pve-qemu-machine'),
 567     arch => {
 568         description => "Virtual processor architecture. Defaults to the host.",
 569         optional => 1,
 570         type => 'string',
 571         enum => [qw(x86_64 aarch64)],
 572     },
 573     smbios1 => {
 574         description => "Specify SMBIOS type 1 fields.",
 575         type => 'string', format => 'pve-qm-smbios1',
 576         maxLength => 256,
 577         optional => 1,
 578     },
 579     protection => {
 580         optional => 1,
 581         type => 'boolean',
 582         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 583         default => 0,
 584     },
 585     bios => {
 586         optional => 1,
 587         type => 'string',
 588         enum => [ qw(seabios ovmf) ],
 589         description => "Select BIOS implementation.",
 590         default => 'seabios',
 591     },
 592     vmgenid => {
 593         type => 'string',
 594         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 595         format_description => 'UUID',
 596         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 597         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 598             " 128-bit integer value identifier to the guest OS. This allows to".
 599             " notify the guest operating system when the virtual machine is".
 600             " executed with a different configuration (e.g. snapshot execution".
 601             " or creation from a template). The guest operating system notices".
 602             " the change, and is then able to react as appropriate by marking".
 603             " its copies of distributed databases as dirty, re-initializing its".
 604             " random number generator, etc.\n".
 605             "Note that auto-creation only works when done throug API/CLI create".
 606             " or update methods, but not when manually editing the config file.",
 607         default => "1 (autogenerated)",
 608         optional => 1,
 609     },
 610 };
 611
 612 my $confdesc_cloudinit = {
 613     citype => {
 614         optional => 1,
 615         type => 'string',
 616         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 617         enum => ['configdrive2', 'nocloud'],
 618     },
 619     ciuser => {
 620         optional => 1,
 621         type => 'string',
 622         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 623     },
 624     cipassword => {
 625         optional => 1,
 626         type => 'string',
 627         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 628     },
 629     searchdomain => {
 630         optional => 1,
 631         type => 'string',
 632         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 633     },
 634     nameserver => {
 635         optional => 1,
 636         type => 'string', format => 'address-list',
 637         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 638     },
 639     sshkeys => {
 640         optional => 1,
 641         type => 'string',
 642         format => 'urlencoded',
 643         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 644     },
 645 };
 646
 647 # what about other qemu settings ?
 648 #cpu => 'string',
 649 #machine => 'string',
 650 #fda => 'file',
 651 #fdb => 'file',
 652 #mtdblock => 'file',
 653 #sd => 'file',
 654 #pflash => 'file',
 655 #snapshot => 'bool',
 656 #bootp => 'file',
 657 ##tftp => 'dir',
 658 ##smb => 'dir',
 659 #kernel => 'file',
 660 #append => 'string',
 661 #initrd => 'file',
 662 ##soundhw => 'string',
 663
 664 while (my ($k, $v) = each %$confdesc) {
 665     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 666 }
 667
 668 my $MAX_IDE_DISKS = 4;
 669 my $MAX_SCSI_DISKS = 14;
 670 my $MAX_VIRTIO_DISKS = 16;
 671 my $MAX_SATA_DISKS = 6;
 672 my $MAX_USB_DEVICES = 5;
 673 my $MAX_NETS = 32;
 674 my $MAX_UNUSED_DISKS = 256;
 675 my $MAX_HOSTPCI_DEVICES = 4;
 676 my $MAX_SERIAL_PORTS = 4;
 677 my $MAX_PARALLEL_PORTS = 3;
 678 my $MAX_NUMA = 8;
 679
 680 my $numa_fmt = {
 681     cpus => {
 682         type => "string",
 683         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 684         description => "CPUs accessing this NUMA node.",
 685         format_description => "id[-id];...",
 686     },
 687     memory => {
 688         type => "number",
 689         description => "Amount of memory this NUMA node provides.",
 690         optional => 1,
 691     },
 692     hostnodes => {
 693         type => "string",
 694         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 695         description => "Host NUMA nodes to use.",
 696         format_description => "id[-id];...",
 697         optional => 1,
 698     },
 699     policy => {
 700         type => 'string',
 701         enum => [qw(preferred bind interleave)],
 702         description => "NUMA allocation policy.",
 703         optional => 1,
 704     },
 705 };
 706 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 707 my $numadesc = {
 708     optional => 1,
 709     type => 'string', format => $numa_fmt,
 710     description => "NUMA topology.",
 711 };
 712 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 713
 714 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 715     $confdesc->{"numa$i"} = $numadesc;
 716 }
 717
 718 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 719                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 720                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 721 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 722
 723 my $net_fmt_bridge_descr = <<__EOD__;
 724 Bridge to attach the network device to. The Proxmox VE standard bridge
 725 is called 'vmbr0'.
 726
 727 If you do not specify a bridge, we create a kvm user (NATed) network
 728 device, which provides DHCP and DNS services. The following addresses
 729 are used:
 730
 731  10.0.2.2   Gateway
 732  10.0.2.3   DNS Server
 733  10.0.2.4   SMB Server
 734
 735 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 736 __EOD__
 737
 738 my $net_fmt = {
 739     macaddr => {
 740         type => 'string',
 741         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 742         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 743         format_description => "XX:XX:XX:XX:XX:XX",
 744         optional => 1,
 745     },
 746     model => {
 747         type => 'string',
 748         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 749         enum => $nic_model_list,
 750         default_key => 1,
 751     },
 752     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 753     bridge => {
 754         type => 'string',
 755         description => $net_fmt_bridge_descr,
 756         format_description => 'bridge',
 757         optional => 1,
 758     },
 759     queues => {
 760         type => 'integer',
 761         minimum => 0, maximum => 16,
 762         description => 'Number of packet queues to be used on the device.',
 763         optional => 1,
 764     },
 765     rate => {
 766         type => 'number',
 767         minimum => 0,
 768         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 769         optional => 1,
 770     },
 771     tag => {
 772         type => 'integer',
 773         minimum => 1, maximum => 4094,
 774         description => 'VLAN tag to apply to packets on this interface.',
 775         optional => 1,
 776     },
 777     trunks => {
 778         type => 'string',
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => 'VLAN trunks to pass through this interface.',
 781         format_description => 'vlanid[;vlanid...]',
 782         optional => 1,
 783     },
 784     firewall => {
 785         type => 'boolean',
 786         description => 'Whether this interface should be protected by the firewall.',
 787         optional => 1,
 788     },
 789     link_down => {
 790         type => 'boolean',
 791         description => 'Whether this interface should be disconnected (like pulling the plug).',
 792         optional => 1,
 793     },
 794 };
 795
 796 my $netdesc = {
 797     optional => 1,
 798     type => 'string', format => $net_fmt,
 799     description => "Specify network devices.",
 800 };
 801
 802 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 803
 804 my $ipconfig_fmt = {
 805     ip => {
 806         type => 'string',
 807         format => 'pve-ipv4-config',
 808         format_description => 'IPv4Format/CIDR',
 809         description => 'IPv4 address in CIDR format.',
 810         optional => 1,
 811         default => 'dhcp',
 812     },
 813     gw => {
 814         type => 'string',
 815         format => 'ipv4',
 816         format_description => 'GatewayIPv4',
 817         description => 'Default gateway for IPv4 traffic.',
 818         optional => 1,
 819         requires => 'ip',
 820     },
 821     ip6 => {
 822         type => 'string',
 823         format => 'pve-ipv6-config',
 824         format_description => 'IPv6Format/CIDR',
 825         description => 'IPv6 address in CIDR format.',
 826         optional => 1,
 827         default => 'dhcp',
 828     },
 829     gw6 => {
 830         type => 'string',
 831         format => 'ipv6',
 832         format_description => 'GatewayIPv6',
 833         description => 'Default gateway for IPv6 traffic.',
 834         optional => 1,
 835         requires => 'ip6',
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 839 my $ipconfigdesc = {
 840     optional => 1,
 841     type => 'string', format => 'pve-qm-ipconfig',
 842     description => <<'EODESCR',
 843 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 844
 845 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 846
 847 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 848 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 849
 850 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 851 EODESCR
 852 };
 853 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 854
 855 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 856     $confdesc->{"net$i"} = $netdesc;
 857     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 858 }
 859
 860 foreach my $key (keys %$confdesc_cloudinit) {
 861     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 862 }
 863
 864 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 865 sub verify_volume_id_or_qm_path {
 866     my ($volid, $noerr) = @_;
 867
 868     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 869         return $volid;
 870     }
 871
 872     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 873     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 874     if ($@) {
 875         return undef if $noerr;
 876         die $@;
 877     }
 878     return $volid;
 879 }
 880
 881 my $drivename_hash;
 882
 883 my %drivedesc_base = (
 884     volume => { alias => 'file' },
 885     file => {
 886         type => 'string',
 887         format => 'pve-volume-id-or-qm-path',
 888         default_key => 1,
 889         format_description => 'volume',
 890         description => "The drive's backing volume.",
 891     },
 892     media => {
 893         type => 'string',
 894         enum => [qw(cdrom disk)],
 895         description => "The drive's media type.",
 896         default => 'disk',
 897         optional => 1
 898     },
 899     cyls => {
 900         type => 'integer',
 901         description => "Force the drive's physical geometry to have a specific cylinder count.",
 902         optional => 1
 903     },
 904     heads => {
 905         type => 'integer',
 906         description => "Force the drive's physical geometry to have a specific head count.",
 907         optional => 1
 908     },
 909     secs => {
 910         type => 'integer',
 911         description => "Force the drive's physical geometry to have a specific sector count.",
 912         optional => 1
 913     },
 914     trans => {
 915         type => 'string',
 916         enum => [qw(none lba auto)],
 917         description => "Force disk geometry bios translation mode.",
 918         optional => 1,
 919     },
 920     snapshot => {
 921         type => 'boolean',
 922         description => "Controls qemu's snapshot mode feature."
 923             . " If activated, changes made to the disk are temporary and will"
 924             . " be discarded when the VM is shutdown.",
 925         optional => 1,
 926     },
 927     cache => {
 928         type => 'string',
 929         enum => [qw(none writethrough writeback unsafe directsync)],
 930         description => "The drive's cache mode",
 931         optional => 1,
 932     },
 933     format => get_standard_option('pve-qm-image-format'),
 934     size => {
 935         type => 'string',
 936         format => 'disk-size',
 937         format_description => 'DiskSize',
 938         description => "Disk size. This is purely informational and has no effect.",
 939         optional => 1,
 940     },
 941     backup => {
 942         type => 'boolean',
 943         description => "Whether the drive should be included when making backups.",
 944         optional => 1,
 945     },
 946     replicate => {
 947         type => 'boolean',
 948         description => 'Whether the drive should considered for replication jobs.',
 949         optional => 1,
 950         default => 1,
 951     },
 952     rerror => {
 953         type => 'string',
 954         enum => [qw(ignore report stop)],
 955         description => 'Read error action.',
 956         optional => 1,
 957     },
 958     werror => {
 959         type => 'string',
 960         enum => [qw(enospc ignore report stop)],
 961         description => 'Write error action.',
 962         optional => 1,
 963     },
 964     aio => {
 965         type => 'string',
 966         enum => [qw(native threads)],
 967         description => 'AIO type to use.',
 968         optional => 1,
 969     },
 970     discard => {
 971         type => 'string',
 972         enum => [qw(ignore on)],
 973         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 974         optional => 1,
 975     },
 976     detect_zeroes => {
 977         type => 'boolean',
 978         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 979         optional => 1,
 980     },
 981     serial => {
 982         type => 'string',
 983         format => 'urlencoded',
 984         format_description => 'serial',
 985         maxLength => 20*3, # *3 since it's %xx url enoded
 986         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 987         optional => 1,
 988     },
 989     shared => {
 990         type => 'boolean',
 991         description => 'Mark this locally-managed volume as available on all nodes',
 992         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 993         optional => 1,
 994         default => 0,
 995     }
 996 );
 997
 998 my %iothread_fmt = ( iothread => {
 999         type => 'boolean',
1000         description => "Whether to use iothreads for this drive",
1001         optional => 1,
1002 });
1003
1004 my %model_fmt = (
1005     model => {
1006         type => 'string',
1007         format => 'urlencoded',
1008         format_description => 'model',
1009         maxLength => 40*3, # *3 since it's %xx url enoded
1010         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1011         optional => 1,
1012     },
1013 );
1014
1015 my %queues_fmt = (
1016     queues => {
1017         type => 'integer',
1018         description => "Number of queues.",
1019         minimum => 2,
1020         optional => 1
1021     }
1022 );
1023
1024 my %scsiblock_fmt = (
1025     scsiblock => {
1026         type => 'boolean',
1027         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1028         optional => 1,
1029         default => 0,
1030     },
1031 );
1032
1033 my %ssd_fmt = (
1034     ssd => {
1035         type => 'boolean',
1036         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1037         optional => 1,
1038     },
1039 );
1040
1041 my $add_throttle_desc = sub {
1042     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1043     my $d = {
1044         type => $type,
1045         format_description => $unit,
1046         description => "Maximum $what in $longunit.",
1047         optional => 1,
1048     };
1049     $d->{minimum} = $minimum if defined($minimum);
1050     $drivedesc_base{$key} = $d;
1051 };
1052 # throughput: (leaky bucket)
1053 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1054 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1056 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1057 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1059 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1060 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1061 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1062
1063 # pools: (pool of IO before throttling starts taking effect)
1064 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1065 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1067 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1068 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1069 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1070
1071 # burst lengths
1072 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1073 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1075 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1078
1079 # legacy support
1080 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1081 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1082 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1083 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1084
1085 my $ide_fmt = {
1086     %drivedesc_base,
1087     %model_fmt,
1088     %ssd_fmt,
1089 };
1090 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1091
1092 my $idedesc = {
1093     optional => 1,
1094     type => 'string', format => $ide_fmt,
1095     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1096 };
1097 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1098
1099 my $scsi_fmt = {
1100     %drivedesc_base,
1101     %iothread_fmt,
1102     %queues_fmt,
1103     %scsiblock_fmt,
1104     %ssd_fmt,
1105 };
1106 my $scsidesc = {
1107     optional => 1,
1108     type => 'string', format => $scsi_fmt,
1109     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1112
1113 my $sata_fmt = {
1114     %drivedesc_base,
1115     %ssd_fmt,
1116 };
1117 my $satadesc = {
1118     optional => 1,
1119     type => 'string', format => $sata_fmt,
1120     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1123
1124 my $virtio_fmt = {
1125     %drivedesc_base,
1126     %iothread_fmt,
1127 };
1128 my $virtiodesc = {
1129     optional => 1,
1130     type => 'string', format => $virtio_fmt,
1131     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1132 };
1133 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1134
1135 my $alldrive_fmt = {
1136     %drivedesc_base,
1137     %iothread_fmt,
1138     %model_fmt,
1139     %queues_fmt,
1140     %scsiblock_fmt,
1141     %ssd_fmt,
1142 };
1143
1144 my $efidisk_fmt = {
1145     volume => { alias => 'file' },
1146     file => {
1147         type => 'string',
1148         format => 'pve-volume-id-or-qm-path',
1149         default_key => 1,
1150         format_description => 'volume',
1151         description => "The drive's backing volume.",
1152     },
1153     format => get_standard_option('pve-qm-image-format'),
1154     size => {
1155         type => 'string',
1156         format => 'disk-size',
1157         format_description => 'DiskSize',
1158         description => "Disk size. This is purely informational and has no effect.",
1159         optional => 1,
1160     },
1161 };
1162
1163 my $efidisk_desc = {
1164     optional => 1,
1165     type => 'string', format => $efidisk_fmt,
1166     description => "Configure a Disk for storing EFI vars",
1167 };
1168
1169 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1170
1171 my $usb_fmt = {
1172     host => {
1173         default_key => 1,
1174         type => 'string', format => 'pve-qm-usb-device',
1175         format_description => 'HOSTUSBDEVICE|spice',
1176         description => <<EODESCR,
1177 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1178
1179  'bus-port(.port)*' (decimal numbers) or
1180  'vendor_id:product_id' (hexadeciaml numbers) or
1181  'spice'
1182
1183 You can use the 'lsusb -t' command to list existing usb devices.
1184
1185 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1186
1187 The value 'spice' can be used to add a usb redirection devices for spice.
1188 EODESCR
1189     },
1190     usb3 => {
1191         optional => 1,
1192         type => 'boolean',
1193         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1194         default => 0,
1195     },
1196 };
1197
1198 my $usbdesc = {
1199     optional => 1,
1200     type => 'string', format => $usb_fmt,
1201     description => "Configure an USB device (n is 0 to 4).",
1202 };
1203 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1204
1205 # NOTE: the match-groups of this regex are used in parse_hostpci
1206 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1207 my $hostpci_fmt = {
1208     host => {
1209         default_key => 1,
1210         type => 'string',
1211         pattern => qr/$PCIRE(;$PCIRE)*/,
1212         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1213         description => <<EODESCR,
1214 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1215 of PCI virtual functions of the host. HOSTPCIID syntax is:
1216
1217 'bus:dev.func' (hexadecimal numbers)
1218
1219 You can us the 'lspci' command to list existing PCI devices.
1220 EODESCR
1221     },
1222     rombar => {
1223         type => 'boolean',
1224         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1225         optional => 1,
1226         default => 1,
1227     },
1228     romfile => {
1229         type => 'string',
1230         pattern => '[^,;]+',
1231         format_description => 'string',
1232         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1233         optional => 1,
1234     },
1235     pcie => {
1236         type => 'boolean',
1237         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1238         optional => 1,
1239         default => 0,
1240     },
1241     'x-vga' => {
1242         type => 'boolean',
1243         description =>  "Enable vfio-vga device support.",
1244         optional => 1,
1245         default => 0,
1246     },
1247     'mdev' => {
1248         type => 'string',
1249         format_description => 'string',
1250         pattern => '[^/\.:]+',
1251         optional => 1,
1252         description => <<EODESCR
1253 The type of mediated device to use.
1254 An instance of this type will be created on startup of the VM and
1255 will be cleaned up when the VM stops.
1256 EODESCR
1257     }
1258 };
1259 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1260
1261 my $hostpcidesc = {
1262         optional => 1,
1263         type => 'string', format => 'pve-qm-hostpci',
1264         description => "Map host PCI devices into guest.",
1265         verbose_description =>  <<EODESCR,
1266 Map host PCI devices into guest.
1267
1268 NOTE: This option allows direct access to host hardware. So it is no longer
1269 possible to migrate such machines - use with special care.
1270
1271 CAUTION: Experimental! User reported problems with this option.
1272 EODESCR
1273 };
1274 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1275
1276 my $serialdesc = {
1277         optional => 1,
1278         type => 'string',
1279         pattern => '(/dev/.+|socket)',
1280         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1281         verbose_description =>  <<EODESCR,
1282 Create a serial device inside the VM (n is 0 to 3), and pass through a
1283 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1284 host side (use 'qm terminal' to open a terminal connection).
1285
1286 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1287
1288 CAUTION: Experimental! User reported problems with this option.
1289 EODESCR
1290 };
1291
1292 my $paralleldesc= {
1293         optional => 1,
1294         type => 'string',
1295         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1296         description =>  "Map host parallel devices (n is 0 to 2).",
1297         verbose_description =>  <<EODESCR,
1298 Map host parallel devices (n is 0 to 2).
1299
1300 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1301
1302 CAUTION: Experimental! User reported problems with this option.
1303 EODESCR
1304 };
1305
1306 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1307     $confdesc->{"parallel$i"} = $paralleldesc;
1308 }
1309
1310 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1311     $confdesc->{"serial$i"} = $serialdesc;
1312 }
1313
1314 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1315     $confdesc->{"hostpci$i"} = $hostpcidesc;
1316 }
1317
1318 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1319     $drivename_hash->{"ide$i"} = 1;
1320     $confdesc->{"ide$i"} = $idedesc;
1321 }
1322
1323 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1324     $drivename_hash->{"sata$i"} = 1;
1325     $confdesc->{"sata$i"} = $satadesc;
1326 }
1327
1328 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1329     $drivename_hash->{"scsi$i"} = 1;
1330     $confdesc->{"scsi$i"} = $scsidesc ;
1331 }
1332
1333 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1334     $drivename_hash->{"virtio$i"} = 1;
1335     $confdesc->{"virtio$i"} = $virtiodesc;
1336 }
1337
1338 $drivename_hash->{efidisk0} = 1;
1339 $confdesc->{efidisk0} = $efidisk_desc;
1340
1341 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1342     $confdesc->{"usb$i"} = $usbdesc;
1343 }
1344
1345 my $unuseddesc = {
1346     optional => 1,
1347     type => 'string', format => 'pve-volume-id',
1348     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1349 };
1350
1351 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1352     $confdesc->{"unused$i"} = $unuseddesc;
1353 }
1354
1355 my $kvm_api_version = 0;
1356
1357 sub kvm_version {
1358     return $kvm_api_version if $kvm_api_version;
1359
1360     open my $fh, '<', '/dev/kvm'
1361         or return undef;
1362
1363     # 0xae00 => KVM_GET_API_VERSION
1364     $kvm_api_version = ioctl($fh, 0xae00, 0);
1365
1366     return $kvm_api_version;
1367 }
1368
1369 my $kvm_user_version;
1370
1371 sub kvm_user_version {
1372
1373     return $kvm_user_version if $kvm_user_version;
1374
1375     $kvm_user_version = 'unknown';
1376
1377     my $code = sub {
1378         my $line = shift;
1379         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1380             $kvm_user_version = $2;
1381         }
1382     };
1383
1384     eval { run_command("kvm -version", outfunc => $code); };
1385     warn $@ if $@;
1386
1387     return $kvm_user_version;
1388
1389 }
1390
1391 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1392
1393 sub valid_drive_names {
1394     # order is important - used to autoselect boot disk
1395     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1396             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1397             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1398             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1399             'efidisk0');
1400 }
1401
1402 sub is_valid_drivename {
1403     my $dev = shift;
1404
1405     return defined($drivename_hash->{$dev});
1406 }
1407
1408 sub option_exists {
1409     my $key = shift;
1410     return defined($confdesc->{$key});
1411 }
1412
1413 sub nic_models {
1414     return $nic_model_list;
1415 }
1416
1417 sub os_list_description {
1418
1419     return {
1420         other => 'Other',
1421         wxp => 'Windows XP',
1422         w2k => 'Windows 2000',
1423         w2k3 =>, 'Windows 2003',
1424         w2k8 => 'Windows 2008',
1425         wvista => 'Windows Vista',
1426         win7 => 'Windows 7',
1427         win8 => 'Windows 8/2012',
1428         win10 => 'Windows 10/2016',
1429         l24 => 'Linux 2.4',
1430         l26 => 'Linux 2.6',
1431     };
1432 }
1433
1434 my $cdrom_path;
1435
1436 sub get_cdrom_path {
1437
1438     return  $cdrom_path if $cdrom_path;
1439
1440     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1441     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1442     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1443 }
1444
1445 sub get_iso_path {
1446     my ($storecfg, $vmid, $cdrom) = @_;
1447
1448     if ($cdrom eq 'cdrom') {
1449         return get_cdrom_path();
1450     } elsif ($cdrom eq 'none') {
1451         return '';
1452     } elsif ($cdrom =~ m|^/|) {
1453         return $cdrom;
1454     } else {
1455         return PVE::Storage::path($storecfg, $cdrom);
1456     }
1457 }
1458
1459 # try to convert old style file names to volume IDs
1460 sub filename_to_volume_id {
1461     my ($vmid, $file, $media) = @_;
1462
1463      if (!($file eq 'none' || $file eq 'cdrom' ||
1464           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1465
1466         return undef if $file =~ m|/|;
1467
1468         if ($media && $media eq 'cdrom') {
1469             $file = "local:iso/$file";
1470         } else {
1471             $file = "local:$vmid/$file";
1472         }
1473     }
1474
1475     return $file;
1476 }
1477
1478 sub verify_media_type {
1479     my ($opt, $vtype, $media) = @_;
1480
1481     return if !$media;
1482
1483     my $etype;
1484     if ($media eq 'disk') {
1485         $etype = 'images';
1486     } elsif ($media eq 'cdrom') {
1487         $etype = 'iso';
1488     } else {
1489         die "internal error";
1490     }
1491
1492     return if ($vtype eq $etype);
1493
1494     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1495 }
1496
1497 sub cleanup_drive_path {
1498     my ($opt, $storecfg, $drive) = @_;
1499
1500     # try to convert filesystem paths to volume IDs
1501
1502     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1503         ($drive->{file} !~ m|^/dev/.+|) &&
1504         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1505         ($drive->{file} !~ m/^\d+$/)) {
1506         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1507         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1508         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1509         verify_media_type($opt, $vtype, $drive->{media});
1510         $drive->{file} = $volid;
1511     }
1512
1513     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1514 }
1515
1516 sub parse_hotplug_features {
1517     my ($data) = @_;
1518
1519     my $res = {};
1520
1521     return $res if $data eq '0';
1522
1523     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1524
1525     foreach my $feature (PVE::Tools::split_list($data)) {
1526         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1527             $res->{$1} = 1;
1528         } else {
1529             die "invalid hotplug feature '$feature'\n";
1530         }
1531     }
1532     return $res;
1533 }
1534
1535 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1536 sub pve_verify_hotplug_features {
1537     my ($value, $noerr) = @_;
1538
1539     return $value if parse_hotplug_features($value);
1540
1541     return undef if $noerr;
1542
1543     die "unable to parse hotplug option\n";
1544 }
1545
1546 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1547 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1548 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1549 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1550 #        [,iothread=on][,serial=serial][,model=model]
1551
1552 sub parse_drive {
1553     my ($key, $data) = @_;
1554
1555     my ($interface, $index);
1556
1557     if ($key =~ m/^([^\d]+)(\d+)$/) {
1558         $interface = $1;
1559         $index = $2;
1560     } else {
1561         return undef;
1562     }
1563
1564     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1565                                      : $confdesc->{$key}->{format};
1566     if (!$desc) {
1567         warn "invalid drive key: $key\n";
1568         return undef;
1569     }
1570     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1571     return undef if !$res;
1572     $res->{interface} = $interface;
1573     $res->{index} = $index;
1574
1575     my $error = 0;
1576     foreach my $opt (qw(bps bps_rd bps_wr)) {
1577         if (my $bps = defined(delete $res->{$opt})) {
1578             if (defined($res->{"m$opt"})) {
1579                 warn "both $opt and m$opt specified\n";
1580                 ++$error;
1581                 next;
1582             }
1583             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1584         }
1585     }
1586
1587     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1588     for my $requirement (
1589         [mbps_max => 'mbps'],
1590         [mbps_rd_max => 'mbps_rd'],
1591         [mbps_wr_max => 'mbps_wr'],
1592         [miops_max => 'miops'],
1593         [miops_rd_max => 'miops_rd'],
1594         [miops_wr_max => 'miops_wr'],
1595         [bps_max_length => 'mbps_max'],
1596         [bps_rd_max_length => 'mbps_rd_max'],
1597         [bps_wr_max_length => 'mbps_wr_max'],
1598         [iops_max_length => 'iops_max'],
1599         [iops_rd_max_length => 'iops_rd_max'],
1600         [iops_wr_max_length => 'iops_wr_max']) {
1601         my ($option, $requires) = @$requirement;
1602         if ($res->{$option} && !$res->{$requires}) {
1603             warn "$option requires $requires\n";
1604             ++$error;
1605         }
1606     }
1607
1608     return undef if $error;
1609
1610     return undef if $res->{mbps_rd} && $res->{mbps};
1611     return undef if $res->{mbps_wr} && $res->{mbps};
1612     return undef if $res->{iops_rd} && $res->{iops};
1613     return undef if $res->{iops_wr} && $res->{iops};
1614
1615     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1616         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1617         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1618         return undef if $res->{interface} eq 'virtio';
1619     }
1620
1621     if (my $size = $res->{size}) {
1622         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1623     }
1624
1625     return $res;
1626 }
1627
1628 sub print_drive {
1629     my ($vmid, $drive) = @_;
1630     my $data = { %$drive };
1631     delete $data->{$_} for qw(index interface);
1632     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1633 }
1634
1635 sub scsi_inquiry {
1636     my($fh, $noerr) = @_;
1637
1638     my $SG_IO = 0x2285;
1639     my $SG_GET_VERSION_NUM = 0x2282;
1640
1641     my $versionbuf = "\x00" x 8;
1642     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1643     if (!$ret) {
1644         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1645         return undef;
1646     }
1647     my $version = unpack("I", $versionbuf);
1648     if ($version < 30000) {
1649         die "scsi generic interface too old\n"  if !$noerr;
1650         return undef;
1651     }
1652
1653     my $buf = "\x00" x 36;
1654     my $sensebuf = "\x00" x 8;
1655     my $cmd = pack("C x3 C x1", 0x12, 36);
1656
1657     # see /usr/include/scsi/sg.h
1658     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1659
1660     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1661                       length($sensebuf), 0, length($buf), $buf,
1662                       $cmd, $sensebuf, 6000);
1663
1664     $ret = ioctl($fh, $SG_IO, $packet);
1665     if (!$ret) {
1666         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1667         return undef;
1668     }
1669
1670     my @res = unpack($sg_io_hdr_t, $packet);
1671     if ($res[17] || $res[18]) {
1672         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1673         return undef;
1674     }
1675
1676     my $res = {};
1677     (my $byte0, my $byte1, $res->{vendor},
1678      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1679
1680     $res->{removable} = $byte1 & 128 ? 1 : 0;
1681     $res->{type} = $byte0 & 31;
1682
1683     return $res;
1684 }
1685
1686 sub path_is_scsi {
1687     my ($path) = @_;
1688
1689     my $fh = IO::File->new("+<$path") || return undef;
1690     my $res = scsi_inquiry($fh, 1);
1691     close($fh);
1692
1693     return $res;
1694 }
1695
1696 sub machine_type_is_q35 {
1697     my ($conf) = @_;
1698
1699     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1700 }
1701
1702 sub print_tabletdevice_full {
1703     my ($conf, $arch) = @_;
1704
1705     my $q35 = machine_type_is_q35($conf);
1706
1707     # we use uhci for old VMs because tablet driver was buggy in older qemu
1708     my $usbbus;
1709     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1710         $usbbus = 'ehci';
1711     } else {
1712         $usbbus = 'uhci';
1713     }
1714
1715     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1716 }
1717
1718 sub print_keyboarddevice_full {
1719     my ($conf, $arch, $machine) = @_;
1720
1721     return undef if $arch ne 'aarch64';
1722
1723     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1724 }
1725
1726 sub print_drivedevice_full {
1727     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1728
1729     my $device = '';
1730     my $maxdev = 0;
1731
1732     if ($drive->{interface} eq 'virtio') {
1733         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1734         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1735         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1736     } elsif ($drive->{interface} eq 'scsi') {
1737
1738         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1739         my $unit = $drive->{index} % $maxdev;
1740         my $devicetype = 'hd';
1741         my $path = '';
1742         if (drive_is_cdrom($drive)) {
1743             $devicetype = 'cd';
1744         } else {
1745             if ($drive->{file} =~ m|^/|) {
1746                 $path = $drive->{file};
1747                 if (my $info = path_is_scsi($path)) {
1748                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1749                         $devicetype = 'block';
1750                     } elsif ($info->{type} == 1) { # tape
1751                         $devicetype = 'generic';
1752                     }
1753                 }
1754             } else {
1755                  $path = PVE::Storage::path($storecfg, $drive->{file});
1756             }
1757
1758             if($path =~ m/^iscsi\:\/\//){
1759                 $devicetype = 'generic';
1760             }
1761         }
1762
1763         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1764             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1765         } else {
1766             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1767         }
1768
1769         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1770             $device .= ",rotation_rate=1";
1771         }
1772
1773     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1774         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1775         my $controller = int($drive->{index} / $maxdev);
1776         my $unit = $drive->{index} % $maxdev;
1777         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1778
1779         $device = "ide-$devicetype";
1780         if ($drive->{interface} eq 'ide') {
1781             $device .= ",bus=ide.$controller,unit=$unit";
1782         } else {
1783             $device .= ",bus=ahci$controller.$unit";
1784         }
1785         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1786
1787         if ($devicetype eq 'hd') {
1788             if (my $model = $drive->{model}) {
1789                 $model = URI::Escape::uri_unescape($model);
1790                 $device .= ",model=$model";
1791             }
1792             if ($drive->{ssd}) {
1793                 $device .= ",rotation_rate=1";
1794             }
1795         }
1796     } elsif ($drive->{interface} eq 'usb') {
1797         die "implement me";
1798         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1799     } else {
1800         die "unsupported interface type";
1801     }
1802
1803     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1804
1805     if (my $serial = $drive->{serial}) {
1806         $serial = URI::Escape::uri_unescape($serial);
1807         $device .= ",serial=$serial";
1808     }
1809
1810
1811     return $device;
1812 }
1813
1814 sub get_initiator_name {
1815     my $initiator;
1816
1817     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1818     while (defined(my $line = <$fh>)) {
1819         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1820         $initiator = $1;
1821         last;
1822     }
1823     $fh->close();
1824
1825     return $initiator;
1826 }
1827
1828 sub print_drive_full {
1829     my ($storecfg, $vmid, $drive) = @_;
1830
1831     my $path;
1832     my $volid = $drive->{file};
1833     my $format;
1834
1835     if (drive_is_cdrom($drive)) {
1836         $path = get_iso_path($storecfg, $vmid, $volid);
1837     } else {
1838         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1839         if ($storeid) {
1840             $path = PVE::Storage::path($storecfg, $volid);
1841             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1842             $format = qemu_img_format($scfg, $volname);
1843         } else {
1844             $path = $volid;
1845             $format = "raw";
1846         }
1847    }
1848
1849     my $opts = '';
1850     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1851     foreach my $o (@qemu_drive_options) {
1852         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1853     }
1854
1855     # snapshot only accepts on|off
1856     if (defined($drive->{snapshot})) {
1857         my $v = $drive->{snapshot} ? 'on' : 'off';
1858         $opts .= ",snapshot=$v";
1859     }
1860
1861     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1862         my ($dir, $qmpname) = @$type;
1863         if (my $v = $drive->{"mbps$dir"}) {
1864             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1865         }
1866         if (my $v = $drive->{"mbps${dir}_max"}) {
1867             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1868         }
1869         if (my $v = $drive->{"bps${dir}_max_length"}) {
1870             $opts .= ",throttling.bps$qmpname-max-length=$v";
1871         }
1872         if (my $v = $drive->{"iops${dir}"}) {
1873             $opts .= ",throttling.iops$qmpname=$v";
1874         }
1875         if (my $v = $drive->{"iops${dir}_max"}) {
1876             $opts .= ",throttling.iops$qmpname-max=$v";
1877         }
1878         if (my $v = $drive->{"iops${dir}_max_length"}) {
1879             $opts .= ",throttling.iops$qmpname-max-length=$v";
1880         }
1881     }
1882
1883     $opts .= ",format=$format" if $format && !$drive->{format};
1884
1885     my $cache_direct = 0;
1886
1887     if (my $cache = $drive->{cache}) {
1888         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1889     } elsif (!drive_is_cdrom($drive)) {
1890         $opts .= ",cache=none";
1891         $cache_direct = 1;
1892     }
1893
1894     # aio native works only with O_DIRECT
1895     if (!$drive->{aio}) {
1896         if($cache_direct) {
1897             $opts .= ",aio=native";
1898         } else {
1899             $opts .= ",aio=threads";
1900         }
1901     }
1902
1903     if (!drive_is_cdrom($drive)) {
1904         my $detectzeroes;
1905         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1906             $detectzeroes = 'off';
1907         } elsif ($drive->{discard}) {
1908             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1909         } else {
1910             # This used to be our default with discard not being specified:
1911             $detectzeroes = 'on';
1912         }
1913         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1914     }
1915
1916     my $pathinfo = $path ? "file=$path," : '';
1917
1918     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1919 }
1920
1921 sub print_netdevice_full {
1922     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1923
1924     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1925
1926     my $device = $net->{model};
1927     if ($net->{model} eq 'virtio') {
1928          $device = 'virtio-net-pci';
1929      };
1930
1931     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1932     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1933     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1934         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1935         my $vectors = $net->{queues} * 2 + 2;
1936         $tmpstr .= ",vectors=$vectors,mq=on";
1937     }
1938     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1939
1940     if ($use_old_bios_files) {
1941         my $romfile;
1942         if ($device eq 'virtio-net-pci') {
1943             $romfile = 'pxe-virtio.rom';
1944         } elsif ($device eq 'e1000') {
1945             $romfile = 'pxe-e1000.rom';
1946         } elsif ($device eq 'ne2k') {
1947             $romfile = 'pxe-ne2k_pci.rom';
1948         } elsif ($device eq 'pcnet') {
1949             $romfile = 'pxe-pcnet.rom';
1950         } elsif ($device eq 'rtl8139') {
1951             $romfile = 'pxe-rtl8139.rom';
1952         }
1953         $tmpstr .= ",romfile=$romfile" if $romfile;
1954     }
1955
1956     return $tmpstr;
1957 }
1958
1959 sub print_netdev_full {
1960     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1961
1962     my $i = '';
1963     if ($netid =~ m/^net(\d+)$/) {
1964         $i = int($1);
1965     }
1966
1967     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1968
1969     my $ifname = "tap${vmid}i$i";
1970
1971     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1972     die "interface name '$ifname' is too long (max 15 character)\n"
1973         if length($ifname) >= 16;
1974
1975     my $vhostparam = '';
1976     if (is_native($arch)) {
1977         $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1978     }
1979
1980     my $vmname = $conf->{name} || "vm$vmid";
1981
1982     my $netdev = "";
1983     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1984
1985     if ($net->{bridge}) {
1986         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1987     } else {
1988         $netdev = "type=user,id=$netid,hostname=$vmname";
1989     }
1990
1991     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1992
1993     return $netdev;
1994 }
1995
1996
1997 sub print_cpu_device {
1998     my ($conf, $id) = @_;
1999
2000     my $kvm = $conf->{kvm} // 1;
2001     my $cpu = $kvm ? "kvm64" : "qemu64";
2002     if (my $cputype = $conf->{cpu}) {
2003         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2004             or die "Cannot parse cpu description: $cputype\n";
2005         $cpu = $cpuconf->{cputype};
2006     }
2007
2008     my $cores = $conf->{cores} || 1;
2009
2010     my $current_core = ($id - 1) % $cores;
2011     my $current_socket = int(($id - 1 - $current_core)/$cores);
2012
2013     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2014 }
2015
2016 my $vga_map = {
2017     'cirrus' => 'cirrus-vga',
2018     'std' => 'VGA',
2019     'vmware' => 'vmware-svga',
2020     'virtio' => 'virtio-vga',
2021 };
2022
2023 sub print_vga_device {
2024     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2025
2026     my $type = $vga_map->{$vga->{type}};
2027     if ($type eq 'virtio-vga' && $arch eq 'aarch64') {
2028         $type = 'virtio-gpu';
2029     }
2030     my $vgamem_mb = $vga->{memory};
2031     if ($qxlnum) {
2032         $type = $id ? 'qxl' : 'qxl-vga';
2033     }
2034     die "no devicetype for $vga->{type}\n" if !$type;
2035
2036     my $memory = "";
2037     if ($vgamem_mb) {
2038         if ($vga->{type} eq 'virtio') {
2039             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2040             $memory = ",max_hostmem=$bytes";
2041         } elsif ($qxlnum) {
2042             # from https://www.spice-space.org/multiple-monitors.html
2043             $memory = ",vgamem_mb=$vga->{memory}";
2044             my $ram = $vgamem_mb * 4;
2045             my $vram = $vgamem_mb * 2;
2046             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2047         } else {
2048             $memory = ",vgamem_mb=$vga->{memory}";
2049         }
2050     } elsif ($qxlnum && $id) {
2051         $memory = ",ram_size=67108864,vram_size=33554432";
2052     }
2053
2054     my $q35 = machine_type_is_q35($conf);
2055     my $vgaid = "vga" . ($id // '');
2056     my $pciaddr;
2057
2058     if ($q35 && $vgaid eq 'vga') {
2059         # the first display uses pcie.0 bus on q35 machines
2060         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2061     } else {
2062         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2063     }
2064
2065     return "$type,id=${vgaid}${memory}${pciaddr}";
2066 }
2067
2068 sub drive_is_cloudinit {
2069     my ($drive) = @_;
2070     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2071 }
2072
2073 sub drive_is_cdrom {
2074     my ($drive, $exclude_cloudinit) = @_;
2075
2076     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2077
2078     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2079
2080 }
2081
2082 sub parse_number_sets {
2083     my ($set) = @_;
2084     my $res = [];
2085     foreach my $part (split(/;/, $set)) {
2086         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2087             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2088             push @$res, [ $1, $2 ];
2089         } else {
2090             die "invalid range: $part\n";
2091         }
2092     }
2093     return $res;
2094 }
2095
2096 sub parse_numa {
2097     my ($data) = @_;
2098
2099     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2100     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2101     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2102     return $res;
2103 }
2104
2105 sub parse_hostpci {
2106     my ($value) = @_;
2107
2108     return undef if !$value;
2109
2110     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2111
2112     my @idlist = split(/;/, $res->{host});
2113     delete $res->{host};
2114     foreach my $id (@idlist) {
2115         if ($id =~ /^$PCIRE$/) {
2116             if (defined($2)) {
2117                 push @{$res->{pciid}}, { id => $1, function => $2 };
2118             } else {
2119                 my $pcidevices = PVE::SysFSTools::lspci($1);
2120                 $res->{pciid} = $pcidevices->{$1};
2121             }
2122         } else {
2123             # should have been caught by parse_property_string already
2124             die "failed to parse PCI id: $id\n";
2125         }
2126     }
2127     return $res;
2128 }
2129
2130 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2131 sub parse_net {
2132     my ($data) = @_;
2133
2134     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2135     if ($@) {
2136         warn $@;
2137         return undef;
2138     }
2139     if (!defined($res->{macaddr})) {
2140         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2141         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2142     }
2143     return $res;
2144 }
2145
2146 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2147 sub parse_ipconfig {
2148     my ($data) = @_;
2149
2150     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2151     if ($@) {
2152         warn $@;
2153         return undef;
2154     }
2155
2156     if ($res->{gw} && !$res->{ip}) {
2157         warn 'gateway specified without specifying an IP address';
2158         return undef;
2159     }
2160     if ($res->{gw6} && !$res->{ip6}) {
2161         warn 'IPv6 gateway specified without specifying an IPv6 address';
2162         return undef;
2163     }
2164     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2165         warn 'gateway specified together with DHCP';
2166         return undef;
2167     }
2168     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2169         # gw6 + auto/dhcp
2170         warn "IPv6 gateway specified together with $res->{ip6} address";
2171         return undef;
2172     }
2173
2174     if (!$res->{ip} && !$res->{ip6}) {
2175         return { ip => 'dhcp', ip6 => 'dhcp' };
2176     }
2177
2178     return $res;
2179 }
2180
2181 sub print_net {
2182     my $net = shift;
2183
2184     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2185 }
2186
2187 sub add_random_macs {
2188     my ($settings) = @_;
2189
2190     foreach my $opt (keys %$settings) {
2191         next if $opt !~ m/^net(\d+)$/;
2192         my $net = parse_net($settings->{$opt});
2193         next if !$net;
2194         $settings->{$opt} = print_net($net);
2195     }
2196 }
2197
2198 sub vm_is_volid_owner {
2199     my ($storecfg, $vmid, $volid) = @_;
2200
2201     if ($volid !~  m|^/|) {
2202         my ($path, $owner);
2203         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2204         if ($owner && ($owner == $vmid)) {
2205             return 1;
2206         }
2207     }
2208
2209     return undef;
2210 }
2211
2212 sub split_flagged_list {
2213     my $text = shift || '';
2214     $text =~ s/[,;]/ /g;
2215     $text =~ s/^\s+//;
2216     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2217 }
2218
2219 sub join_flagged_list {
2220     my ($how, $lst) = @_;
2221     join $how, map { $lst->{$_} . $_ } keys %$lst;
2222 }
2223
2224 sub vmconfig_delete_pending_option {
2225     my ($conf, $key, $force) = @_;
2226
2227     delete $conf->{pending}->{$key};
2228     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2229     $pending_delete_hash->{$key} = $force ? '!' : '';
2230     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2231 }
2232
2233 sub vmconfig_undelete_pending_option {
2234     my ($conf, $key) = @_;
2235
2236     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2237     delete $pending_delete_hash->{$key};
2238
2239     if (%$pending_delete_hash) {
2240         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2241     } else {
2242         delete $conf->{pending}->{delete};
2243     }
2244 }
2245
2246 sub vmconfig_register_unused_drive {
2247     my ($storecfg, $vmid, $conf, $drive) = @_;
2248
2249     if (drive_is_cloudinit($drive)) {
2250         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2251         warn $@ if $@;
2252     } elsif (!drive_is_cdrom($drive)) {
2253         my $volid = $drive->{file};
2254         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2255             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2256         }
2257     }
2258 }
2259
2260 sub vmconfig_cleanup_pending {
2261     my ($conf) = @_;
2262
2263     # remove pending changes when nothing changed
2264     my $changes;
2265     foreach my $opt (keys %{$conf->{pending}}) {
2266         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2267             $changes = 1;
2268             delete $conf->{pending}->{$opt};
2269         }
2270     }
2271
2272     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2273     my $pending_delete_hash = {};
2274     while (my ($opt, $force) = each %$current_delete_hash) {
2275         if (defined($conf->{$opt})) {
2276             $pending_delete_hash->{$opt} = $force;
2277         } else {
2278             $changes = 1;
2279         }
2280     }
2281
2282     if (%$pending_delete_hash) {
2283         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2284     } else {
2285         delete $conf->{pending}->{delete};
2286     }
2287
2288     return $changes;
2289 }
2290
2291 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2292 my $smbios1_fmt = {
2293     uuid => {
2294         type => 'string',
2295         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2296         format_description => 'UUID',
2297         description => "Set SMBIOS1 UUID.",
2298         optional => 1,
2299     },
2300     version => {
2301         type => 'string',
2302         pattern => '\S+',
2303         format_description => 'string',
2304         description => "Set SMBIOS1 version.",
2305         optional => 1,
2306     },
2307     serial => {
2308         type => 'string',
2309         pattern => '\S+',
2310         format_description => 'string',
2311         description => "Set SMBIOS1 serial number.",
2312         optional => 1,
2313     },
2314     manufacturer => {
2315         type => 'string',
2316         pattern => '\S+',
2317         format_description => 'string',
2318         description => "Set SMBIOS1 manufacturer.",
2319         optional => 1,
2320     },
2321     product => {
2322         type => 'string',
2323         pattern => '\S+',
2324         format_description => 'string',
2325         description => "Set SMBIOS1 product ID.",
2326         optional => 1,
2327     },
2328     sku => {
2329         type => 'string',
2330         pattern => '\S+',
2331         format_description => 'string',
2332         description => "Set SMBIOS1 SKU string.",
2333         optional => 1,
2334     },
2335     family => {
2336         type => 'string',
2337         pattern => '\S+',
2338         format_description => 'string',
2339         description => "Set SMBIOS1 family string.",
2340         optional => 1,
2341     },
2342 };
2343
2344 sub parse_smbios1 {
2345     my ($data) = @_;
2346
2347     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2348     warn $@ if $@;
2349     return $res;
2350 }
2351
2352 sub print_smbios1 {
2353     my ($smbios1) = @_;
2354     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2355 }
2356
2357 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2358
2359 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2360 sub verify_bootdisk {
2361     my ($value, $noerr) = @_;
2362
2363     return $value if is_valid_drivename($value);
2364
2365     return undef if $noerr;
2366
2367     die "invalid boot disk '$value'\n";
2368 }
2369
2370 sub parse_watchdog {
2371     my ($value) = @_;
2372
2373     return undef if !$value;
2374
2375     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2376     warn $@ if $@;
2377     return $res;
2378 }
2379
2380 sub parse_guest_agent {
2381     my ($value) = @_;
2382
2383     return {} if !defined($value->{agent});
2384
2385     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2386     warn $@ if $@;
2387
2388     # if the agent is disabled ignore the other potentially set properties
2389     return {} if !$res->{enabled};
2390     return $res;
2391 }
2392
2393 sub parse_vga {
2394     my ($value) = @_;
2395
2396     return {} if !$value;
2397     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2398     warn $@ if $@;
2399     return $res;
2400 }
2401
2402 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2403 sub verify_usb_device {
2404     my ($value, $noerr) = @_;
2405
2406     return $value if parse_usb_device($value);
2407
2408     return undef if $noerr;
2409
2410     die "unable to parse usb device\n";
2411 }
2412
2413 # add JSON properties for create and set function
2414 sub json_config_properties {
2415     my $prop = shift;
2416
2417     foreach my $opt (keys %$confdesc) {
2418         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2419         $prop->{$opt} = $confdesc->{$opt};
2420     }
2421
2422     return $prop;
2423 }
2424
2425 # return copy of $confdesc_cloudinit to generate documentation
2426 sub cloudinit_config_properties {
2427
2428     return dclone($confdesc_cloudinit);
2429 }
2430
2431 sub check_type {
2432     my ($key, $value) = @_;
2433
2434     die "unknown setting '$key'\n" if !$confdesc->{$key};
2435
2436     my $type = $confdesc->{$key}->{type};
2437
2438     if (!defined($value)) {
2439         die "got undefined value\n";
2440     }
2441
2442     if ($value =~ m/[\n\r]/) {
2443         die "property contains a line feed\n";
2444     }
2445
2446     if ($type eq 'boolean') {
2447         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2448         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2449         die "type check ('boolean') failed - got '$value'\n";
2450     } elsif ($type eq 'integer') {
2451         return int($1) if $value =~ m/^(\d+)$/;
2452         die "type check ('integer') failed - got '$value'\n";
2453     } elsif ($type eq 'number') {
2454         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2455         die "type check ('number') failed - got '$value'\n";
2456     } elsif ($type eq 'string') {
2457         if (my $fmt = $confdesc->{$key}->{format}) {
2458             PVE::JSONSchema::check_format($fmt, $value);
2459             return $value;
2460         }
2461         $value =~ s/^\"(.*)\"$/$1/;
2462         return $value;
2463     } else {
2464         die "internal error"
2465     }
2466 }
2467
2468 sub touch_config {
2469     my ($vmid) = @_;
2470
2471     my $conf = PVE::QemuConfig->config_file($vmid);
2472     utime undef, undef, $conf;
2473 }
2474
2475 sub destroy_vm {
2476     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2477
2478     my $conffile = PVE::QemuConfig->config_file($vmid);
2479
2480     my $conf = PVE::QemuConfig->load_config($vmid);
2481
2482     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2483
2484     if ($conf->{template}) {
2485         # check if any base image is still used by a linked clone
2486         foreach_drive($conf, sub {
2487                 my ($ds, $drive) = @_;
2488
2489                 return if drive_is_cdrom($drive);
2490
2491                 my $volid = $drive->{file};
2492
2493                 return if !$volid || $volid =~ m|^/|;
2494
2495                 die "base volume '$volid' is still in use by linked cloned\n"
2496                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2497
2498         });
2499     }
2500
2501     # only remove disks owned by this VM
2502     foreach_drive($conf, sub {
2503         my ($ds, $drive) = @_;
2504
2505         return if drive_is_cdrom($drive, 1);
2506
2507         my $volid = $drive->{file};
2508
2509         return if !$volid || $volid =~ m|^/|;
2510
2511         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2512         return if !$path || !$owner || ($owner != $vmid);
2513
2514         eval {
2515             PVE::Storage::vdisk_free($storecfg, $volid);
2516         };
2517         warn "Could not remove disk '$volid', check manually: $@" if $@;
2518
2519     });
2520
2521     if ($keep_empty_config) {
2522         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2523     } else {
2524         unlink $conffile;
2525     }
2526
2527     # also remove unused disk
2528     eval {
2529         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2530
2531         eval {
2532             PVE::Storage::foreach_volid($dl, sub {
2533                 my ($volid, $sid, $volname, $d) = @_;
2534                 PVE::Storage::vdisk_free($storecfg, $volid);
2535             });
2536         };
2537         warn $@ if $@;
2538
2539     };
2540     warn $@ if $@;
2541 }
2542
2543 sub parse_vm_config {
2544     my ($filename, $raw) = @_;
2545
2546     return undef if !defined($raw);
2547
2548     my $res = {
2549         digest => Digest::SHA::sha1_hex($raw),
2550         snapshots => {},
2551         pending => {},
2552     };
2553
2554     $filename =~ m|/qemu-server/(\d+)\.conf$|
2555         || die "got strange filename '$filename'";
2556
2557     my $vmid = $1;
2558
2559     my $conf = $res;
2560     my $descr;
2561     my $section = '';
2562
2563     my @lines = split(/\n/, $raw);
2564     foreach my $line (@lines) {
2565         next if $line =~ m/^\s*$/;
2566
2567         if ($line =~ m/^\[PENDING\]\s*$/i) {
2568             $section = 'pending';
2569             if (defined($descr)) {
2570                 $descr =~ s/\s+$//;
2571                 $conf->{description} = $descr;
2572             }
2573             $descr = undef;
2574             $conf = $res->{$section} = {};
2575             next;
2576
2577         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2578             $section = $1;
2579             if (defined($descr)) {
2580                 $descr =~ s/\s+$//;
2581                 $conf->{description} = $descr;
2582             }
2583             $descr = undef;
2584             $conf = $res->{snapshots}->{$section} = {};
2585             next;
2586         }
2587
2588         if ($line =~ m/^\#(.*)\s*$/) {
2589             $descr = '' if !defined($descr);
2590             $descr .= PVE::Tools::decode_text($1) . "\n";
2591             next;
2592         }
2593
2594         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2595             $descr = '' if !defined($descr);
2596             $descr .= PVE::Tools::decode_text($2);
2597         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2598             $conf->{snapstate} = $1;
2599         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2600             my $key = $1;
2601             my $value = $2;
2602             $conf->{$key} = $value;
2603         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2604             my $value = $1;
2605             if ($section eq 'pending') {
2606                 $conf->{delete} = $value; # we parse this later
2607             } else {
2608                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2609             }
2610         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2611             my $key = $1;
2612             my $value = $2;
2613             eval { $value = check_type($key, $value); };
2614             if ($@) {
2615                 warn "vm $vmid - unable to parse value of '$key' - $@";
2616             } else {
2617                 $key = 'ide2' if $key eq 'cdrom';
2618                 my $fmt = $confdesc->{$key}->{format};
2619                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2620                     my $v = parse_drive($key, $value);
2621                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2622                         $v->{file} = $volid;
2623                         $value = print_drive($vmid, $v);
2624                     } else {
2625                         warn "vm $vmid - unable to parse value of '$key'\n";
2626                         next;
2627                     }
2628                 }
2629
2630                 $conf->{$key} = $value;
2631             }
2632         }
2633     }
2634
2635     if (defined($descr)) {
2636         $descr =~ s/\s+$//;
2637         $conf->{description} = $descr;
2638     }
2639     delete $res->{snapstate}; # just to be sure
2640
2641     return $res;
2642 }
2643
2644 sub write_vm_config {
2645     my ($filename, $conf) = @_;
2646
2647     delete $conf->{snapstate}; # just to be sure
2648
2649     if ($conf->{cdrom}) {
2650         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2651         $conf->{ide2} = $conf->{cdrom};
2652         delete $conf->{cdrom};
2653     }
2654
2655     # we do not use 'smp' any longer
2656     if ($conf->{sockets}) {
2657         delete $conf->{smp};
2658     } elsif ($conf->{smp}) {
2659         $conf->{sockets} = $conf->{smp};
2660         delete $conf->{cores};
2661         delete $conf->{smp};
2662     }
2663
2664     my $used_volids = {};
2665
2666     my $cleanup_config = sub {
2667         my ($cref, $pending, $snapname) = @_;
2668
2669         foreach my $key (keys %$cref) {
2670             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2671                 $key eq 'snapstate' || $key eq 'pending';
2672             my $value = $cref->{$key};
2673             if ($key eq 'delete') {
2674                 die "propertry 'delete' is only allowed in [PENDING]\n"
2675                     if !$pending;
2676                 # fixme: check syntax?
2677                 next;
2678             }
2679             eval { $value = check_type($key, $value); };
2680             die "unable to parse value of '$key' - $@" if $@;
2681
2682             $cref->{$key} = $value;
2683
2684             if (!$snapname && is_valid_drivename($key)) {
2685                 my $drive = parse_drive($key, $value);
2686                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2687             }
2688         }
2689     };
2690
2691     &$cleanup_config($conf);
2692
2693     &$cleanup_config($conf->{pending}, 1);
2694
2695     foreach my $snapname (keys %{$conf->{snapshots}}) {
2696         die "internal error" if $snapname eq 'pending';
2697         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2698     }
2699
2700     # remove 'unusedX' settings if we re-add a volume
2701     foreach my $key (keys %$conf) {
2702         my $value = $conf->{$key};
2703         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2704             delete $conf->{$key};
2705         }
2706     }
2707
2708     my $generate_raw_config = sub {
2709         my ($conf, $pending) = @_;
2710
2711         my $raw = '';
2712
2713         # add description as comment to top of file
2714         if (defined(my $descr = $conf->{description})) {
2715             if ($descr) {
2716                 foreach my $cl (split(/\n/, $descr)) {
2717                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2718                 }
2719             } else {
2720                 $raw .= "#\n" if $pending;
2721             }
2722         }
2723
2724         foreach my $key (sort keys %$conf) {
2725             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2726             $raw .= "$key: $conf->{$key}\n";
2727         }
2728         return $raw;
2729     };
2730
2731     my $raw = &$generate_raw_config($conf);
2732
2733     if (scalar(keys %{$conf->{pending}})){
2734         $raw .= "\n[PENDING]\n";
2735         $raw .= &$generate_raw_config($conf->{pending}, 1);
2736     }
2737
2738     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2739         $raw .= "\n[$snapname]\n";
2740         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2741     }
2742
2743     return $raw;
2744 }
2745
2746 sub load_defaults {
2747
2748     my $res = {};
2749
2750     # we use static defaults from our JSON schema configuration
2751     foreach my $key (keys %$confdesc) {
2752         if (defined(my $default = $confdesc->{$key}->{default})) {
2753             $res->{$key} = $default;
2754         }
2755     }
2756
2757     return $res;
2758 }
2759
2760 sub config_list {
2761     my $vmlist = PVE::Cluster::get_vmlist();
2762     my $res = {};
2763     return $res if !$vmlist || !$vmlist->{ids};
2764     my $ids = $vmlist->{ids};
2765
2766     foreach my $vmid (keys %$ids) {
2767         my $d = $ids->{$vmid};
2768         next if !$d->{node} || $d->{node} ne $nodename;
2769         next if !$d->{type} || $d->{type} ne 'qemu';
2770         $res->{$vmid}->{exists} = 1;
2771     }
2772     return $res;
2773 }
2774
2775 # test if VM uses local resources (to prevent migration)
2776 sub check_local_resources {
2777     my ($conf, $noerr) = @_;
2778
2779     my $loc_res = 0;
2780
2781     $loc_res = 1 if $conf->{hostusb}; # old syntax
2782     $loc_res = 1 if $conf->{hostpci}; # old syntax
2783
2784     foreach my $k (keys %$conf) {
2785         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2786         # sockets are safe: they will recreated be on the target side post-migrate
2787         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2788         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2789     }
2790
2791     die "VM uses local resources\n" if $loc_res && !$noerr;
2792
2793     return $loc_res;
2794 }
2795
2796 # check if used storages are available on all nodes (use by migrate)
2797 sub check_storage_availability {
2798     my ($storecfg, $conf, $node) = @_;
2799
2800     foreach_drive($conf, sub {
2801         my ($ds, $drive) = @_;
2802
2803         my $volid = $drive->{file};
2804         return if !$volid;
2805
2806         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2807         return if !$sid;
2808
2809         # check if storage is available on both nodes
2810         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2811         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2812    });
2813 }
2814
2815 # list nodes where all VM images are available (used by has_feature API)
2816 sub shared_nodes {
2817     my ($conf, $storecfg) = @_;
2818
2819     my $nodelist = PVE::Cluster::get_nodelist();
2820     my $nodehash = { map { $_ => 1 } @$nodelist };
2821     my $nodename = PVE::INotify::nodename();
2822
2823     foreach_drive($conf, sub {
2824         my ($ds, $drive) = @_;
2825
2826         my $volid = $drive->{file};
2827         return if !$volid;
2828
2829         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2830         if ($storeid) {
2831             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2832             if ($scfg->{disable}) {
2833                 $nodehash = {};
2834             } elsif (my $avail = $scfg->{nodes}) {
2835                 foreach my $node (keys %$nodehash) {
2836                     delete $nodehash->{$node} if !$avail->{$node};
2837                 }
2838             } elsif (!$scfg->{shared}) {
2839                 foreach my $node (keys %$nodehash) {
2840                     delete $nodehash->{$node} if $node ne $nodename
2841                 }
2842             }
2843         }
2844     });
2845
2846     return $nodehash
2847 }
2848
2849 sub check_cmdline {
2850     my ($pidfile, $pid) = @_;
2851
2852     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2853     if (defined($fh)) {
2854         my $line = <$fh>;
2855         $fh->close;
2856         return undef if !$line;
2857         my @param = split(/\0/, $line);
2858
2859         my $cmd = $param[0];
2860         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2861
2862         for (my $i = 0; $i < scalar (@param); $i++) {
2863             my $p = $param[$i];
2864             next if !$p;
2865             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2866                 my $p = $param[$i+1];
2867                 return 1 if $p && ($p eq $pidfile);
2868                 return undef;
2869             }
2870         }
2871     }
2872     return undef;
2873 }
2874
2875 sub check_running {
2876     my ($vmid, $nocheck, $node) = @_;
2877
2878     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2879
2880     die "unable to find configuration file for VM $vmid - no such machine\n"
2881         if !$nocheck && ! -f $filename;
2882
2883     my $pidfile = pidfile_name($vmid);
2884
2885     if (my $fd = IO::File->new("<$pidfile")) {
2886         my $st = stat($fd);
2887         my $line = <$fd>;
2888         close($fd);
2889
2890         my $mtime = $st->mtime;
2891         if ($mtime > time()) {
2892             warn "file '$filename' modified in future\n";
2893         }
2894
2895         if ($line =~ m/^(\d+)$/) {
2896             my $pid = $1;
2897             if (check_cmdline($pidfile, $pid)) {
2898                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2899                     return $pid;
2900                 }
2901             }
2902         }
2903     }
2904
2905     return undef;
2906 }
2907
2908 sub vzlist {
2909
2910     my $vzlist = config_list();
2911
2912     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2913
2914     while (defined(my $de = $fd->read)) {
2915         next if $de !~ m/^(\d+)\.pid$/;
2916         my $vmid = $1;
2917         next if !defined($vzlist->{$vmid});
2918         if (my $pid = check_running($vmid)) {
2919             $vzlist->{$vmid}->{pid} = $pid;
2920         }
2921     }
2922
2923     return $vzlist;
2924 }
2925
2926 sub disksize {
2927     my ($storecfg, $conf) = @_;
2928
2929     my $bootdisk = $conf->{bootdisk};
2930     return undef if !$bootdisk;
2931     return undef if !is_valid_drivename($bootdisk);
2932
2933     return undef if !$conf->{$bootdisk};
2934
2935     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2936     return undef if !defined($drive);
2937
2938     return undef if drive_is_cdrom($drive);
2939
2940     my $volid = $drive->{file};
2941     return undef if !$volid;
2942
2943     return $drive->{size};
2944 }
2945
2946 our $vmstatus_return_properties = {
2947     vmid => get_standard_option('pve-vmid'),
2948     status => {
2949         description => "Qemu process status.",
2950         type => 'string',
2951         enum => ['stopped', 'running'],
2952     },
2953     maxmem => {
2954         description => "Maximum memory in bytes.",
2955         type => 'integer',
2956         optional => 1,
2957         renderer => 'bytes',
2958     },
2959     maxdisk => {
2960         description => "Root disk size in bytes.",
2961         type => 'integer',
2962         optional => 1,
2963         renderer => 'bytes',
2964     },
2965     name => {
2966         description => "VM name.",
2967         type => 'string',
2968         optional => 1,
2969     },
2970     qmpstatus => {
2971         description => "Qemu QMP agent status.",
2972         type => 'string',
2973         optional => 1,
2974     },
2975     pid => {
2976         description => "PID of running qemu process.",
2977         type => 'integer',
2978         optional => 1,
2979     },
2980     uptime => {
2981         description => "Uptime.",
2982         type => 'integer',
2983         optional => 1,
2984         renderer => 'duration',
2985     },
2986     cpus => {
2987         description => "Maximum usable CPUs.",
2988         type => 'number',
2989         optional => 1,
2990     },
2991 };
2992
2993 my $last_proc_pid_stat;
2994
2995 # get VM status information
2996 # This must be fast and should not block ($full == false)
2997 # We only query KVM using QMP if $full == true (this can be slow)
2998 sub vmstatus {
2999     my ($opt_vmid, $full) = @_;
3000
3001     my $res = {};
3002
3003     my $storecfg = PVE::Storage::config();
3004
3005     my $list = vzlist();
3006     my $defaults = load_defaults();
3007
3008     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3009
3010     my $cpucount = $cpuinfo->{cpus} || 1;
3011
3012     foreach my $vmid (keys %$list) {
3013         next if $opt_vmid && ($vmid ne $opt_vmid);
3014
3015         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3016         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3017
3018         my $d = { vmid => $vmid };
3019         $d->{pid} = $list->{$vmid}->{pid};
3020
3021         # fixme: better status?
3022         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3023
3024         my $size = disksize($storecfg, $conf);
3025         if (defined($size)) {
3026             $d->{disk} = 0; # no info available
3027             $d->{maxdisk} = $size;
3028         } else {
3029             $d->{disk} = 0;
3030             $d->{maxdisk} = 0;
3031         }
3032
3033         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3034             * ($conf->{cores} || $defaults->{cores});
3035         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3036         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3037
3038         $d->{name} = $conf->{name} || "VM $vmid";
3039         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3040             : $defaults->{memory}*(1024*1024);
3041
3042         if ($conf->{balloon}) {
3043             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3044             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3045                 : $defaults->{shares};
3046         }
3047
3048         $d->{uptime} = 0;
3049         $d->{cpu} = 0;
3050         $d->{mem} = 0;
3051
3052         $d->{netout} = 0;
3053         $d->{netin} = 0;
3054
3055         $d->{diskread} = 0;
3056         $d->{diskwrite} = 0;
3057
3058         $d->{template} = PVE::QemuConfig->is_template($conf);
3059
3060         $d->{serial} = 1 if conf_has_serial($conf);
3061
3062         $res->{$vmid} = $d;
3063     }
3064
3065     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3066     foreach my $dev (keys %$netdev) {
3067         next if $dev !~ m/^tap([1-9]\d*)i/;
3068         my $vmid = $1;
3069         my $d = $res->{$vmid};
3070         next if !$d;
3071
3072         $d->{netout} += $netdev->{$dev}->{receive};
3073         $d->{netin} += $netdev->{$dev}->{transmit};
3074
3075         if ($full) {
3076             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3077             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3078         }
3079
3080     }
3081
3082     my $ctime = gettimeofday;
3083
3084     foreach my $vmid (keys %$list) {
3085
3086         my $d = $res->{$vmid};
3087         my $pid = $d->{pid};
3088         next if !$pid;
3089
3090         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3091         next if !$pstat; # not running
3092
3093         my $used = $pstat->{utime} + $pstat->{stime};
3094
3095         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3096
3097         if ($pstat->{vsize}) {
3098             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3099         }
3100
3101         my $old = $last_proc_pid_stat->{$pid};
3102         if (!$old) {
3103             $last_proc_pid_stat->{$pid} = {
3104                 time => $ctime,
3105                 used => $used,
3106                 cpu => 0,
3107             };
3108             next;
3109         }
3110
3111         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3112
3113         if ($dtime > 1000) {
3114             my $dutime = $used -  $old->{used};
3115
3116             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3117             $last_proc_pid_stat->{$pid} = {
3118                 time => $ctime,
3119                 used => $used,
3120                 cpu => $d->{cpu},
3121             };
3122         } else {
3123             $d->{cpu} = $old->{cpu};
3124         }
3125     }
3126
3127     return $res if !$full;
3128
3129     my $qmpclient = PVE::QMPClient->new();
3130
3131     my $ballooncb = sub {
3132         my ($vmid, $resp) = @_;
3133
3134         my $info = $resp->{'return'};
3135         return if !$info->{max_mem};
3136
3137         my $d = $res->{$vmid};
3138
3139         # use memory assigned to VM
3140         $d->{maxmem} = $info->{max_mem};
3141         $d->{balloon} = $info->{actual};
3142
3143         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3144             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3145             $d->{freemem} = $info->{free_mem};
3146         }
3147
3148         $d->{ballooninfo} = $info;
3149     };
3150
3151     my $blockstatscb = sub {
3152         my ($vmid, $resp) = @_;
3153         my $data = $resp->{'return'} || [];
3154         my $totalrdbytes = 0;
3155         my $totalwrbytes = 0;
3156
3157         for my $blockstat (@$data) {
3158             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3159             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3160
3161             $blockstat->{device} =~ s/drive-//;
3162             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3163         }
3164         $res->{$vmid}->{diskread} = $totalrdbytes;
3165         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3166     };
3167
3168     my $statuscb = sub {
3169         my ($vmid, $resp) = @_;
3170
3171         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3172         # this fails if ballon driver is not loaded, so this must be
3173         # the last commnand (following command are aborted if this fails).
3174         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3175
3176         my $status = 'unknown';
3177         if (!defined($status = $resp->{'return'}->{status})) {
3178             warn "unable to get VM status\n";
3179             return;
3180         }
3181
3182         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3183     };
3184
3185     foreach my $vmid (keys %$list) {
3186         next if $opt_vmid && ($vmid ne $opt_vmid);
3187         next if !$res->{$vmid}->{pid}; # not running
3188         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3189     }
3190
3191     $qmpclient->queue_execute(undef, 2);
3192
3193     foreach my $vmid (keys %$list) {
3194         next if $opt_vmid && ($vmid ne $opt_vmid);
3195         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3196     }
3197
3198     return $res;
3199 }
3200
3201 sub foreach_drive {
3202     my ($conf, $func, @param) = @_;
3203
3204     foreach my $ds (valid_drive_names()) {
3205         next if !defined($conf->{$ds});
3206
3207         my $drive = parse_drive($ds, $conf->{$ds});
3208         next if !$drive;
3209
3210         &$func($ds, $drive, @param);
3211     }
3212 }
3213
3214 sub foreach_volid {
3215     my ($conf, $func, @param) = @_;
3216
3217     my $volhash = {};
3218
3219     my $test_volid = sub {
3220         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3221
3222         return if !$volid;
3223
3224         $volhash->{$volid}->{cdrom} //= 1;
3225         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3226
3227         $volhash->{$volid}->{replicate} //= 0;
3228         $volhash->{$volid}->{replicate} = 1 if $replicate;
3229
3230         $volhash->{$volid}->{shared} //= 0;
3231         $volhash->{$volid}->{shared} = 1 if $shared;
3232
3233         $volhash->{$volid}->{referenced_in_config} //= 0;
3234         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3235
3236         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3237             if defined($snapname);
3238     };
3239
3240     foreach_drive($conf, sub {
3241         my ($ds, $drive) = @_;
3242         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3243     });
3244
3245     foreach my $snapname (keys %{$conf->{snapshots}}) {
3246         my $snap = $conf->{snapshots}->{$snapname};
3247         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3248         foreach_drive($snap, sub {
3249             my ($ds, $drive) = @_;
3250             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3251         });
3252     }
3253
3254     foreach my $volid (keys %$volhash) {
3255         &$func($volid, $volhash->{$volid}, @param);
3256     }
3257 }
3258
3259 sub conf_has_serial {
3260     my ($conf) = @_;
3261
3262     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3263         if ($conf->{"serial$i"}) {
3264             return 1;
3265         }
3266     }
3267
3268     return 0;
3269 }
3270
3271 sub vga_conf_has_spice {
3272     my ($vga) = @_;
3273
3274     my $vgaconf = parse_vga($vga);
3275     my $vgatype = $vgaconf->{type};
3276     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3277
3278     return $1 || 1;
3279 }
3280
3281 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3282 sub get_host_arch() {
3283     $host_arch = (POSIX::uname())[4] if !$host_arch;
3284     return $host_arch;
3285 }
3286
3287 sub is_native($) {
3288     my ($arch) = @_;
3289     return get_host_arch() eq $arch;
3290 }
3291
3292 my $default_machines = {
3293     x86_64 => 'pc',
3294     aarch64 => 'virt',
3295 };
3296
3297 sub get_basic_machine_info {
3298     my ($conf, $forcemachine) = @_;
3299
3300     my $arch = $conf->{arch} // get_host_arch();
3301     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3302     return ($arch, $machine);
3303 }
3304
3305 sub get_ovmf_files($) {
3306     my ($arch) = @_;
3307
3308     my $ovmf = $OVMF->{$arch}
3309         or die "no OVMF images known for architecture '$arch'\n";
3310
3311     return @$ovmf;
3312 }
3313
3314 my $Arch2Qemu = {
3315     aarch64 => '/usr/bin/qemu-system-aarch64',
3316     x86_64 => '/usr/bin/qemu-system-x86_64',
3317 };
3318 sub get_command_for_arch($) {
3319     my ($arch) = @_;
3320     return '/usr/bin/kvm' if is_native($arch);
3321
3322     my $cmd = $Arch2Qemu->{$arch}
3323         or die "don't know how to emulate architecture '$arch'\n";
3324     return $cmd;
3325 }
3326
3327 sub get_cpu_options {
3328     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3329
3330     my $cpuFlags = [];
3331     my $ostype = $conf->{ostype};
3332
3333     my $cpu = $kvm ? "kvm64" : "qemu64";
3334     if ($arch eq 'aarch64') {
3335         $cpu = 'cortex-a57';
3336     }
3337     if (my $cputype = $conf->{cpu}) {
3338         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3339             or die "Cannot parse cpu description: $cputype\n";
3340         $cpu = $cpuconf->{cputype};
3341         $kvm_off = 1 if $cpuconf->{hidden};
3342
3343         if (defined(my $flags = $cpuconf->{flags})) {
3344             push @$cpuFlags, split(";", $flags);
3345         }
3346     }
3347
3348     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3349
3350     push @$cpuFlags , '-x2apic'
3351         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3352
3353     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3354
3355     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3356
3357     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3358
3359         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3360         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3361     }
3362
3363     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3364
3365     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3366
3367     push @$cpuFlags, 'kvm=off' if $kvm_off;
3368
3369     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3370         push @$cpuFlags, "vendor=${cpu_vendor}"
3371             if $cpu_vendor ne 'default';
3372     } elsif ($arch ne 'aarch64') {
3373         die "internal error"; # should not happen
3374     }
3375
3376     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3377
3378     return ('-cpu', $cpu);
3379 }
3380
3381 sub config_to_command {
3382     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3383
3384     my $cmd = [];
3385     my $globalFlags = [];
3386     my $machineFlags = [];
3387     my $rtcFlags = [];
3388     my $devices = [];
3389     my $pciaddr = '';
3390     my $bridges = {};
3391     my $kvmver = kvm_user_version();
3392     my $vernum = 0; # unknown
3393     my $ostype = $conf->{ostype};
3394     my $winversion = windows_version($ostype);
3395     my $kvm = $conf->{kvm};
3396
3397     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3398     $kvm //= 1 if is_native($arch);
3399
3400     if ($kvm) {
3401         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3402             if !defined kvm_version();
3403     }
3404
3405     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3406         $vernum = $1*1000000+$2*1000;
3407     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3408         $vernum = $1*1000000+$2*1000+$3;
3409     }
3410
3411     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3412
3413     my $have_ovz = -f '/proc/vz/vestat';
3414
3415     my $q35 = machine_type_is_q35($conf);
3416     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3417     my $use_old_bios_files = undef;
3418     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3419
3420     my $cpuunits = defined($conf->{cpuunits}) ?
3421             $conf->{cpuunits} : $defaults->{cpuunits};
3422
3423     push @$cmd, get_command_for_arch($arch);
3424
3425     push @$cmd, '-id', $vmid;
3426
3427     my $vmname = $conf->{name} || "vm$vmid";
3428
3429     push @$cmd, '-name', $vmname;
3430
3431     my $use_virtio = 0;
3432
3433     my $qmpsocket = qmp_socket($vmid);
3434     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3435     push @$cmd, '-mon', "chardev=qmp,mode=control";
3436
3437     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3438         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3439         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3440     }
3441
3442     push @$cmd, '-pidfile' , pidfile_name($vmid);
3443
3444     push @$cmd, '-daemonize';
3445
3446     if ($conf->{smbios1}) {
3447         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3448     }
3449
3450     if ($conf->{vmgenid}) {
3451         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3452     }
3453
3454     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3455     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3456         die "uefi base image not found\n" if ! -f $ovmf_code;
3457
3458         my $path;
3459         my $format;
3460         if (my $efidisk = $conf->{efidisk0}) {
3461             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3462             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3463             $format = $d->{format};
3464             if ($storeid) {
3465                 $path = PVE::Storage::path($storecfg, $d->{file});
3466                 if (!defined($format)) {
3467                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3468                     $format = qemu_img_format($scfg, $volname);
3469                 }
3470             } else {
3471                 $path = $d->{file};
3472                 die "efidisk format must be specified\n"
3473                     if !defined($format);
3474             }
3475         } else {
3476             warn "no efidisk configured! Using temporary efivars disk.\n";
3477             $path = "/tmp/$vmid-ovmf.fd";
3478             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3479             $format = 'raw';
3480         }
3481
3482         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3483         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3484     }
3485
3486
3487     # add usb controllers
3488     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3489     push @$devices, @usbcontrollers if @usbcontrollers;
3490     my $vga = parse_vga($conf->{vga});
3491
3492     my $qxlnum = vga_conf_has_spice($conf->{vga});
3493     $vga->{type} = 'qxl' if $qxlnum;
3494
3495     if (!$vga->{type}) {
3496         if ($arch eq 'aarch64') {
3497             $vga->{type} = 'virtio';
3498         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3499             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3500         } else {
3501             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3502         }
3503     }
3504
3505     # enable absolute mouse coordinates (needed by vnc)
3506     my $tablet;
3507     if (defined($conf->{tablet})) {
3508         $tablet = $conf->{tablet};
3509     } else {
3510         $tablet = $defaults->{tablet};
3511         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3512         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3513     }
3514
3515     if ($tablet) {
3516         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3517         my $kbd = print_keyboarddevice_full($conf, $arch);
3518         push @$devices, '-device', $kbd if defined($kbd);
3519     }
3520
3521     my $kvm_off = 0;
3522     my $gpu_passthrough;
3523
3524     # host pci devices
3525     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3526         my $d = parse_hostpci($conf->{"hostpci$i"});
3527         next if !$d;
3528
3529         my $pcie = $d->{pcie};
3530         if($pcie){
3531             die "q35 machine model is not enabled" if !$q35;
3532             $pciaddr = print_pcie_addr("hostpci$i");
3533         }else{
3534             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3535         }
3536
3537         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3538         my $romfile = $d->{romfile};
3539
3540         my $xvga = '';
3541         if ($d->{'x-vga'}) {
3542             $xvga = ',x-vga=on';
3543             $kvm_off = 1;
3544             $vga->{type} = 'none';
3545             $gpu_passthrough = 1;
3546
3547             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3548                 $xvga = "";
3549             }
3550         }
3551         my $pcidevices = $d->{pciid};
3552         my $multifunction = 1 if @$pcidevices > 1;
3553         my $sysfspath;
3554         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3555             my $id = $pcidevices->[0]->{id};
3556             my $function = $pcidevices->[0]->{function};
3557             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3558             $sysfspath = "/sys/bus/pci/devices/0000:$id.$function/$uuid";
3559         } elsif ($d->{mdev}) {
3560             warn "ignoring mediated device with multifunction device\n";
3561         }
3562
3563         my $j=0;
3564         foreach my $pcidevice (@$pcidevices) {
3565
3566             my $id = "hostpci$i";
3567             $id .= ".$j" if $multifunction;
3568             my $addr = $pciaddr;
3569             $addr .= ".$j" if $multifunction;
3570             my $devicestr = "vfio-pci";
3571             if ($sysfspath) {
3572                 $devicestr .= ",sysfsdev=$sysfspath";
3573             } else {
3574                 $devicestr .= ",host=$pcidevice->{id}.$pcidevice->{function}";
3575             }
3576             $devicestr .= ",id=$id$addr";
3577
3578             if($j == 0){
3579                 $devicestr .= "$rombar$xvga";
3580                 $devicestr .= ",multifunction=on" if $multifunction;
3581                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3582             }
3583
3584             push @$devices, '-device', $devicestr;
3585             $j++;
3586         }
3587     }
3588
3589     # usb devices
3590     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3591     push @$devices, @usbdevices if @usbdevices;
3592     # serial devices
3593     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3594         if (my $path = $conf->{"serial$i"}) {
3595             if ($path eq 'socket') {
3596                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3597                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3598                 # On aarch64, serial0 is the UART device. Qemu only allows
3599                 # connecting UART devices via the '-serial' command line, as
3600                 # the device has a fixed slot on the hardware...
3601                 if ($arch eq 'aarch64' && $i == 0) {
3602                     push @$devices, '-serial', "chardev:serial$i";
3603                 } else {
3604                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3605                 }
3606             } else {
3607                 die "no such serial device\n" if ! -c $path;
3608                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3609                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3610             }
3611         }
3612     }
3613
3614     # parallel devices
3615     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3616         if (my $path = $conf->{"parallel$i"}) {
3617             die "no such parallel device\n" if ! -c $path;
3618             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3619             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3620             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3621         }
3622     }
3623
3624
3625     my $sockets = 1;
3626     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3627     $sockets = $conf->{sockets} if  $conf->{sockets};
3628
3629     my $cores = $conf->{cores} || 1;
3630
3631     my $maxcpus = $sockets * $cores;
3632
3633     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3634
3635     my $allowed_vcpus = $cpuinfo->{cpus};
3636
3637     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3638         if ($allowed_vcpus < $maxcpus);
3639
3640     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3641
3642         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3643         for (my $i = 2; $i <= $vcpus; $i++)  {
3644             my $cpustr = print_cpu_device($conf,$i);
3645             push @$cmd, '-device', $cpustr;
3646         }
3647
3648     } else {
3649
3650         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3651     }
3652     push @$cmd, '-nodefaults';
3653
3654     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3655
3656     my $bootindex_hash = {};
3657     my $i = 1;
3658     foreach my $o (split(//, $bootorder)) {
3659         $bootindex_hash->{$o} = $i*100;
3660         $i++;
3661     }
3662
3663     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3664
3665     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3666
3667     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3668
3669     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3670         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3671         my $socket = vnc_socket($vmid);
3672         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3673     } else {
3674         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3675         push @$cmd, '-nographic';
3676     }
3677
3678     # time drift fix
3679     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3680
3681     my $useLocaltime = $conf->{localtime};
3682
3683     if ($winversion >= 5) { # windows
3684         $useLocaltime = 1 if !defined($conf->{localtime});
3685
3686         # use time drift fix when acpi is enabled
3687         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3688             $tdf = 1 if !defined($conf->{tdf});
3689         }
3690     }
3691
3692     if ($winversion >= 6) {
3693         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3694         push @$cmd, '-no-hpet';
3695     }
3696
3697     push @$rtcFlags, 'driftfix=slew' if $tdf;
3698
3699     if (!$kvm) {
3700         push @$machineFlags, 'accel=tcg';
3701     }
3702
3703     if ($machine_type) {
3704         push @$machineFlags, "type=${machine_type}";
3705     }
3706
3707     if ($conf->{startdate}) {
3708         push @$rtcFlags, "base=$conf->{startdate}";
3709     } elsif ($useLocaltime) {
3710         push @$rtcFlags, 'base=localtime';
3711     }
3712
3713     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3714
3715     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3716
3717     push @$cmd, '-S' if $conf->{freeze};
3718
3719     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3720
3721     # enable sound
3722     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3723     #push @$cmd, '-soundhw', 'es1370';
3724     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3725
3726     if (parse_guest_agent($conf)->{enabled}) {
3727         my $qgasocket = qmp_socket($vmid, 1);
3728         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3729         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3730         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3731         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3732     }
3733
3734     my $spice_port;
3735
3736     if ($qxlnum) {
3737         if ($qxlnum > 1) {
3738             if ($winversion){
3739                 for(my $i = 1; $i < $qxlnum; $i++){
3740                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3741                 }
3742             } else {
3743                 # assume other OS works like Linux
3744                 my ($ram, $vram) = ("134217728", "67108864");
3745                 if ($vga->{memory}) {
3746                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3747                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3748                 }
3749                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3750                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3751             }
3752         }
3753
3754         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3755
3756         my $nodename = PVE::INotify::nodename();
3757         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3758         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3759         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3760         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3761         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3762
3763         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3764
3765         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3766         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3767         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3768     }
3769
3770     # enable balloon by default, unless explicitly disabled
3771     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3772         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3773         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3774     }
3775
3776     if ($conf->{watchdog}) {
3777         my $wdopts = parse_watchdog($conf->{watchdog});
3778         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3779         my $watchdog = $wdopts->{model} || 'i6300esb';
3780         push @$devices, '-device', "$watchdog$pciaddr";
3781         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3782     }
3783
3784     my $vollist = [];
3785     my $scsicontroller = {};
3786     my $ahcicontroller = {};
3787     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3788
3789     # Add iscsi initiator name if available
3790     if (my $initiator = get_initiator_name()) {
3791         push @$devices, '-iscsi', "initiator-name=$initiator";
3792     }
3793
3794     foreach_drive($conf, sub {
3795         my ($ds, $drive) = @_;
3796
3797         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3798             push @$vollist, $drive->{file};
3799         }
3800
3801         # ignore efidisk here, already added in bios/fw handling code above
3802         return if $drive->{interface} eq 'efidisk';
3803
3804         $use_virtio = 1 if $ds =~ m/^virtio/;
3805
3806         if (drive_is_cdrom ($drive)) {
3807             if ($bootindex_hash->{d}) {
3808                 $drive->{bootindex} = $bootindex_hash->{d};
3809                 $bootindex_hash->{d} += 1;
3810             }
3811         } else {
3812             if ($bootindex_hash->{c}) {
3813                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3814                 $bootindex_hash->{c} += 1;
3815             }
3816         }
3817
3818         if($drive->{interface} eq 'virtio'){
3819            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3820         }
3821
3822         if ($drive->{interface} eq 'scsi') {
3823
3824             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3825
3826             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3827             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3828
3829             my $iothread = '';
3830             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3831                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3832                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3833             } elsif ($drive->{iothread}) {
3834                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3835             }
3836
3837             my $queues = '';
3838             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3839                 $queues = ",num_queues=$drive->{queues}";
3840             }
3841
3842             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3843             $scsicontroller->{$controller}=1;
3844         }
3845
3846         if ($drive->{interface} eq 'sata') {
3847            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3848            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3849            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3850            $ahcicontroller->{$controller}=1;
3851         }
3852
3853         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3854         push @$devices, '-drive',$drive_cmd;
3855         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3856     });
3857
3858     for (my $i = 0; $i < $MAX_NETS; $i++) {
3859          next if !$conf->{"net$i"};
3860          my $d = parse_net($conf->{"net$i"});
3861          next if !$d;
3862
3863          $use_virtio = 1 if $d->{model} eq 'virtio';
3864
3865          if ($bootindex_hash->{n}) {
3866             $d->{bootindex} = $bootindex_hash->{n};
3867             $bootindex_hash->{n} += 1;
3868          }
3869
3870          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3871          push @$devices, '-netdev', $netdevfull;
3872
3873          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3874          push @$devices, '-device', $netdevicefull;
3875     }
3876
3877     if (!$q35) {
3878         # add pci bridges
3879         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3880            $bridges->{1} = 1;
3881            $bridges->{2} = 1;
3882         }
3883
3884         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3885
3886         while (my ($k, $v) = each %$bridges) {
3887             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3888             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3889         }
3890     }
3891
3892     # add custom args
3893     if ($conf->{args}) {
3894         my $aa = PVE::Tools::split_args($conf->{args});
3895         push @$cmd, @$aa;
3896     }
3897
3898     push @$cmd, @$devices;
3899     push @$cmd, '-rtc', join(',', @$rtcFlags)
3900         if scalar(@$rtcFlags);
3901     push @$cmd, '-machine', join(',', @$machineFlags)
3902         if scalar(@$machineFlags);
3903     push @$cmd, '-global', join(',', @$globalFlags)
3904         if scalar(@$globalFlags);
3905
3906     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3907 }
3908
3909 sub vnc_socket {
3910     my ($vmid) = @_;
3911     return "${var_run_tmpdir}/$vmid.vnc";
3912 }
3913
3914 sub spice_port {
3915     my ($vmid) = @_;
3916
3917     my $res = vm_mon_cmd($vmid, 'query-spice');
3918
3919     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3920 }
3921
3922 sub qmp_socket {
3923     my ($vmid, $qga, $name) = @_;
3924     my $sockettype = $qga ? 'qga' : 'qmp';
3925     my $ext = $name ? '-'.$name : '';
3926     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3927 }
3928
3929 sub pidfile_name {
3930     my ($vmid) = @_;
3931     return "${var_run_tmpdir}/$vmid.pid";
3932 }
3933
3934 sub vm_devices_list {
3935     my ($vmid) = @_;
3936
3937     my $res = vm_mon_cmd($vmid, 'query-pci');
3938     my $devices_to_check = [];
3939     my $devices = {};
3940     foreach my $pcibus (@$res) {
3941         push @$devices_to_check, @{$pcibus->{devices}},
3942     }
3943
3944     while (@$devices_to_check) {
3945         my $to_check = [];
3946         for my $d (@$devices_to_check) {
3947             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3948             next if !$d->{'pci_bridge'};
3949
3950             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3951             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3952         }
3953         $devices_to_check = $to_check;
3954     }
3955
3956     my $resblock = vm_mon_cmd($vmid, 'query-block');
3957     foreach my $block (@$resblock) {
3958         if($block->{device} =~ m/^drive-(\S+)/){
3959                 $devices->{$1} = 1;
3960         }
3961     }
3962
3963     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3964     foreach my $mice (@$resmice) {
3965         if ($mice->{name} eq 'QEMU HID Tablet') {
3966             $devices->{tablet} = 1;
3967             last;
3968         }
3969     }
3970
3971     # for usb devices there is no query-usb
3972     # but we can iterate over the entries in
3973     # qom-list path=/machine/peripheral
3974     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3975     foreach my $per (@$resperipheral) {
3976         if ($per->{name} =~ m/^usb\d+$/) {
3977             $devices->{$per->{name}} = 1;
3978         }
3979     }
3980
3981     return $devices;
3982 }
3983
3984 sub vm_deviceplug {
3985     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3986
3987     my $q35 = machine_type_is_q35($conf);
3988
3989     my $devices_list = vm_devices_list($vmid);
3990     return 1 if defined($devices_list->{$deviceid});
3991
3992     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3993
3994     if ($deviceid eq 'tablet') {
3995
3996         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3997
3998     } elsif ($deviceid eq 'keyboard') {
3999
4000         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4001
4002     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4003
4004         die "usb hotplug currently not reliable\n";
4005         # since we can't reliably hot unplug all added usb devices
4006         # and usb passthrough disables live migration
4007         # we disable usb hotplugging for now
4008         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4009
4010     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4011
4012         qemu_iothread_add($vmid, $deviceid, $device);
4013
4014         qemu_driveadd($storecfg, $vmid, $device);
4015         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4016
4017         qemu_deviceadd($vmid, $devicefull);
4018         eval { qemu_deviceaddverify($vmid, $deviceid); };
4019         if (my $err = $@) {
4020             eval { qemu_drivedel($vmid, $deviceid); };
4021             warn $@ if $@;
4022             die $err;
4023         }
4024
4025     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4026
4027
4028         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4029         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4030         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4031
4032         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4033
4034         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4035             qemu_iothread_add($vmid, $deviceid, $device);
4036             $devicefull .= ",iothread=iothread-$deviceid";
4037         }
4038
4039         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4040             $devicefull .= ",num_queues=$device->{queues}";
4041         }
4042
4043         qemu_deviceadd($vmid, $devicefull);
4044         qemu_deviceaddverify($vmid, $deviceid);
4045
4046     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4047
4048         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4049         qemu_driveadd($storecfg, $vmid, $device);
4050
4051         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4052         eval { qemu_deviceadd($vmid, $devicefull); };
4053         if (my $err = $@) {
4054             eval { qemu_drivedel($vmid, $deviceid); };
4055             warn $@ if $@;
4056             die $err;
4057         }
4058
4059     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4060
4061         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4062
4063         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4064         my $use_old_bios_files = undef;
4065         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4066
4067         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4068         qemu_deviceadd($vmid, $netdevicefull);
4069         eval { qemu_deviceaddverify($vmid, $deviceid); };
4070         if (my $err = $@) {
4071             eval { qemu_netdevdel($vmid, $deviceid); };
4072             warn $@ if $@;
4073             die $err;
4074         }
4075
4076     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4077
4078         my $bridgeid = $2;
4079         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4080         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4081
4082         qemu_deviceadd($vmid, $devicefull);
4083         qemu_deviceaddverify($vmid, $deviceid);
4084
4085     } else {
4086         die "can't hotplug device '$deviceid'\n";
4087     }
4088
4089     return 1;
4090 }
4091
4092 # fixme: this should raise exceptions on error!
4093 sub vm_deviceunplug {
4094     my ($vmid, $conf, $deviceid) = @_;
4095
4096     my $devices_list = vm_devices_list($vmid);
4097     return 1 if !defined($devices_list->{$deviceid});
4098
4099     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4100
4101     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4102
4103         qemu_devicedel($vmid, $deviceid);
4104
4105     } elsif ($deviceid =~ m/^usb\d+$/) {
4106
4107         die "usb hotplug currently not reliable\n";
4108         # when unplugging usb devices this way,
4109         # there may be remaining usb controllers/hubs
4110         # so we disable it for now
4111         qemu_devicedel($vmid, $deviceid);
4112         qemu_devicedelverify($vmid, $deviceid);
4113
4114     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4115
4116         qemu_devicedel($vmid, $deviceid);
4117         qemu_devicedelverify($vmid, $deviceid);
4118         qemu_drivedel($vmid, $deviceid);
4119         qemu_iothread_del($conf, $vmid, $deviceid);
4120
4121     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4122
4123         qemu_devicedel($vmid, $deviceid);
4124         qemu_devicedelverify($vmid, $deviceid);
4125         qemu_iothread_del($conf, $vmid, $deviceid);
4126
4127     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4128
4129         qemu_devicedel($vmid, $deviceid);
4130         qemu_drivedel($vmid, $deviceid);
4131         qemu_deletescsihw($conf, $vmid, $deviceid);
4132
4133     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4134
4135         qemu_devicedel($vmid, $deviceid);
4136         qemu_devicedelverify($vmid, $deviceid);
4137         qemu_netdevdel($vmid, $deviceid);
4138
4139     } else {
4140         die "can't unplug device '$deviceid'\n";
4141     }
4142
4143     return 1;
4144 }
4145
4146 sub qemu_deviceadd {
4147     my ($vmid, $devicefull) = @_;
4148
4149     $devicefull = "driver=".$devicefull;
4150     my %options =  split(/[=,]/, $devicefull);
4151
4152     vm_mon_cmd($vmid, "device_add" , %options);
4153 }
4154
4155 sub qemu_devicedel {
4156     my ($vmid, $deviceid) = @_;
4157
4158     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4159 }
4160
4161 sub qemu_iothread_add {
4162     my($vmid, $deviceid, $device) = @_;
4163
4164     if ($device->{iothread}) {
4165         my $iothreads = vm_iothreads_list($vmid);
4166         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4167     }
4168 }
4169
4170 sub qemu_iothread_del {
4171     my($conf, $vmid, $deviceid) = @_;
4172
4173     my $device = parse_drive($deviceid, $conf->{$deviceid});
4174     if ($device->{iothread}) {
4175         my $iothreads = vm_iothreads_list($vmid);
4176         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4177     }
4178 }
4179
4180 sub qemu_objectadd {
4181     my($vmid, $objectid, $qomtype) = @_;
4182
4183     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4184
4185     return 1;
4186 }
4187
4188 sub qemu_objectdel {
4189     my($vmid, $objectid) = @_;
4190
4191     vm_mon_cmd($vmid, "object-del", id => $objectid);
4192
4193     return 1;
4194 }
4195
4196 sub qemu_driveadd {
4197     my ($storecfg, $vmid, $device) = @_;
4198
4199     my $drive = print_drive_full($storecfg, $vmid, $device);
4200     $drive =~ s/\\/\\\\/g;
4201     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4202
4203     # If the command succeeds qemu prints: "OK"
4204     return 1 if $ret =~ m/OK/s;
4205
4206     die "adding drive failed: $ret\n";
4207 }
4208
4209 sub qemu_drivedel {
4210     my($vmid, $deviceid) = @_;
4211
4212     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4213     $ret =~ s/^\s+//;
4214
4215     return 1 if $ret eq "";
4216
4217     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4218     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4219
4220     die "deleting drive $deviceid failed : $ret\n";
4221 }
4222
4223 sub qemu_deviceaddverify {
4224     my ($vmid, $deviceid) = @_;
4225
4226     for (my $i = 0; $i <= 5; $i++) {
4227          my $devices_list = vm_devices_list($vmid);
4228          return 1 if defined($devices_list->{$deviceid});
4229          sleep 1;
4230     }
4231
4232     die "error on hotplug device '$deviceid'\n";
4233 }
4234
4235
4236 sub qemu_devicedelverify {
4237     my ($vmid, $deviceid) = @_;
4238
4239     # need to verify that the device is correctly removed as device_del
4240     # is async and empty return is not reliable
4241
4242     for (my $i = 0; $i <= 5; $i++) {
4243          my $devices_list = vm_devices_list($vmid);
4244          return 1 if !defined($devices_list->{$deviceid});
4245          sleep 1;
4246     }
4247
4248     die "error on hot-unplugging device '$deviceid'\n";
4249 }
4250
4251 sub qemu_findorcreatescsihw {
4252     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4253
4254     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4255
4256     my $scsihwid="$controller_prefix$controller";
4257     my $devices_list = vm_devices_list($vmid);
4258
4259     if(!defined($devices_list->{$scsihwid})) {
4260         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4261     }
4262
4263     return 1;
4264 }
4265
4266 sub qemu_deletescsihw {
4267     my ($conf, $vmid, $opt) = @_;
4268
4269     my $device = parse_drive($opt, $conf->{$opt});
4270
4271     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4272         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4273         return 1;
4274     }
4275
4276     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4277
4278     my $devices_list = vm_devices_list($vmid);
4279     foreach my $opt (keys %{$devices_list}) {
4280         if (PVE::QemuServer::is_valid_drivename($opt)) {
4281             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4282             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4283                 return 1;
4284             }
4285         }
4286     }
4287
4288     my $scsihwid="scsihw$controller";
4289
4290     vm_deviceunplug($vmid, $conf, $scsihwid);
4291
4292     return 1;
4293 }
4294
4295 sub qemu_add_pci_bridge {
4296     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4297
4298     my $bridges = {};
4299
4300     my $bridgeid;
4301
4302     print_pci_addr($device, $bridges, $arch, $machine_type);
4303
4304     while (my ($k, $v) = each %$bridges) {
4305         $bridgeid = $k;
4306     }
4307     return 1 if !defined($bridgeid) || $bridgeid < 1;
4308
4309     my $bridge = "pci.$bridgeid";
4310     my $devices_list = vm_devices_list($vmid);
4311
4312     if (!defined($devices_list->{$bridge})) {
4313         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4314     }
4315
4316     return 1;
4317 }
4318
4319 sub qemu_set_link_status {
4320     my ($vmid, $device, $up) = @_;
4321
4322     vm_mon_cmd($vmid, "set_link", name => $device,
4323                up => $up ? JSON::true : JSON::false);
4324 }
4325
4326 sub qemu_netdevadd {
4327     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4328
4329     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4330     my %options =  split(/[=,]/, $netdev);
4331
4332     vm_mon_cmd($vmid, "netdev_add",  %options);
4333     return 1;
4334 }
4335
4336 sub qemu_netdevdel {
4337     my ($vmid, $deviceid) = @_;
4338
4339     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4340 }
4341
4342 sub qemu_usb_hotplug {
4343     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4344
4345     return if !$device;
4346
4347     # remove the old one first
4348     vm_deviceunplug($vmid, $conf, $deviceid);
4349
4350     # check if xhci controller is necessary and available
4351     if ($device->{usb3}) {
4352
4353         my $devicelist = vm_devices_list($vmid);
4354
4355         if (!$devicelist->{xhci}) {
4356             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4357             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4358         }
4359     }
4360     my $d = parse_usb_device($device->{host});
4361     $d->{usb3} = $device->{usb3};
4362
4363     # add the new one
4364     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4365 }
4366
4367 sub qemu_cpu_hotplug {
4368     my ($vmid, $conf, $vcpus) = @_;
4369
4370     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4371
4372     my $sockets = 1;
4373     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4374     $sockets = $conf->{sockets} if  $conf->{sockets};
4375     my $cores = $conf->{cores} || 1;
4376     my $maxcpus = $sockets * $cores;
4377
4378     $vcpus = $maxcpus if !$vcpus;
4379
4380     die "you can't add more vcpus than maxcpus\n"
4381         if $vcpus > $maxcpus;
4382
4383     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4384
4385     if ($vcpus < $currentvcpus) {
4386
4387         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4388
4389             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4390                 qemu_devicedel($vmid, "cpu$i");
4391                 my $retry = 0;
4392                 my $currentrunningvcpus = undef;
4393                 while (1) {
4394                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4395                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4396                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4397                     $retry++;
4398                     sleep 1;
4399                 }
4400                 #update conf after each succesfull cpu unplug
4401                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4402                 PVE::QemuConfig->write_config($vmid, $conf);
4403             }
4404         } else {
4405             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4406         }
4407
4408         return;
4409     }
4410
4411     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4412     die "vcpus in running vm does not match its configuration\n"
4413         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4414
4415     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4416
4417         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4418             my $cpustr = print_cpu_device($conf, $i);
4419             qemu_deviceadd($vmid, $cpustr);
4420
4421             my $retry = 0;
4422             my $currentrunningvcpus = undef;
4423             while (1) {
4424                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4425                 last if scalar(@{$currentrunningvcpus}) == $i;
4426                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4427                 sleep 1;
4428                 $retry++;
4429             }
4430             #update conf after each succesfull cpu hotplug
4431             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4432             PVE::QemuConfig->write_config($vmid, $conf);
4433         }
4434     } else {
4435
4436         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4437             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4438         }
4439     }
4440 }
4441
4442 sub qemu_block_set_io_throttle {
4443     my ($vmid, $deviceid,
4444         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4445         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4446         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4447         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4448
4449     return if !check_running($vmid) ;
4450
4451     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4452         bps => int($bps),
4453         bps_rd => int($bps_rd),
4454         bps_wr => int($bps_wr),
4455         iops => int($iops),
4456         iops_rd => int($iops_rd),
4457         iops_wr => int($iops_wr),
4458         bps_max => int($bps_max),
4459         bps_rd_max => int($bps_rd_max),
4460         bps_wr_max => int($bps_wr_max),
4461         iops_max => int($iops_max),
4462         iops_rd_max => int($iops_rd_max),
4463         iops_wr_max => int($iops_wr_max),
4464         bps_max_length => int($bps_max_length),
4465         bps_rd_max_length => int($bps_rd_max_length),
4466         bps_wr_max_length => int($bps_wr_max_length),
4467         iops_max_length => int($iops_max_length),
4468         iops_rd_max_length => int($iops_rd_max_length),
4469         iops_wr_max_length => int($iops_wr_max_length),
4470     );
4471
4472 }
4473
4474 # old code, only used to shutdown old VM after update
4475 sub __read_avail {
4476     my ($fh, $timeout) = @_;
4477
4478     my $sel = new IO::Select;
4479     $sel->add($fh);
4480
4481     my $res = '';
4482     my $buf;
4483
4484     my @ready;
4485     while (scalar (@ready = $sel->can_read($timeout))) {
4486         my $count;
4487         if ($count = $fh->sysread($buf, 8192)) {
4488             if ($buf =~ /^(.*)\(qemu\) $/s) {
4489                 $res .= $1;
4490                 last;
4491             } else {
4492                 $res .= $buf;
4493             }
4494         } else {
4495             if (!defined($count)) {
4496                 die "$!\n";
4497             }
4498             last;
4499         }
4500     }
4501
4502     die "monitor read timeout\n" if !scalar(@ready);
4503
4504     return $res;
4505 }
4506
4507 sub qemu_block_resize {
4508     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4509
4510     my $running = check_running($vmid);
4511
4512     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4513
4514     return if !$running;
4515
4516     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4517
4518 }
4519
4520 sub qemu_volume_snapshot {
4521     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4522
4523     my $running = check_running($vmid);
4524
4525     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4526         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4527     } else {
4528         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4529     }
4530 }
4531
4532 sub qemu_volume_snapshot_delete {
4533     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4534
4535     my $running = check_running($vmid);
4536
4537     if($running) {
4538
4539         $running = undef;
4540         my $conf = PVE::QemuConfig->load_config($vmid);
4541         foreach_drive($conf, sub {
4542             my ($ds, $drive) = @_;
4543             $running = 1 if $drive->{file} eq $volid;
4544         });
4545     }
4546
4547     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4548         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4549     } else {
4550         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4551     }
4552 }
4553
4554 sub set_migration_caps {
4555     my ($vmid) = @_;
4556
4557     my $cap_ref = [];
4558
4559     my $enabled_cap = {
4560         "auto-converge" => 1,
4561         "xbzrle" => 1,
4562         "x-rdma-pin-all" => 0,
4563         "zero-blocks" => 0,
4564         "compress" => 0
4565     };
4566
4567     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4568
4569     for my $supported_capability (@$supported_capabilities) {
4570         push @$cap_ref, {
4571             capability => $supported_capability->{capability},
4572             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4573         };
4574     }
4575
4576     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4577 }
4578
4579 my $fast_plug_option = {
4580     'lock' => 1,
4581     'name' => 1,
4582     'onboot' => 1,
4583     'shares' => 1,
4584     'startup' => 1,
4585     'description' => 1,
4586     'protection' => 1,
4587     'vmstatestorage' => 1,
4588 };
4589
4590 # hotplug changes in [PENDING]
4591 # $selection hash can be used to only apply specified options, for
4592 # example: { cores => 1 } (only apply changed 'cores')
4593 # $errors ref is used to return error messages
4594 sub vmconfig_hotplug_pending {
4595     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4596
4597     my $defaults = load_defaults();
4598     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4599
4600     # commit values which do not have any impact on running VM first
4601     # Note: those option cannot raise errors, we we do not care about
4602     # $selection and always apply them.
4603
4604     my $add_error = sub {
4605         my ($opt, $msg) = @_;
4606         $errors->{$opt} = "hotplug problem - $msg";
4607     };
4608
4609     my $changes = 0;
4610     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4611         if ($fast_plug_option->{$opt}) {
4612             $conf->{$opt} = $conf->{pending}->{$opt};
4613             delete $conf->{pending}->{$opt};
4614             $changes = 1;
4615         }
4616     }
4617
4618     if ($changes) {
4619         PVE::QemuConfig->write_config($vmid, $conf);
4620         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4621     }
4622
4623     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4624
4625     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4626     while (my ($opt, $force) = each %$pending_delete_hash) {
4627         next if $selection && !$selection->{$opt};
4628         eval {
4629             if ($opt eq 'hotplug') {
4630                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4631             } elsif ($opt eq 'tablet') {
4632                 die "skip\n" if !$hotplug_features->{usb};
4633                 if ($defaults->{tablet}) {
4634                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4635                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4636                         if $arch eq 'aarch64';
4637                 } else {
4638                     vm_deviceunplug($vmid, $conf, 'tablet');
4639                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4640                 }
4641             } elsif ($opt =~ m/^usb\d+/) {
4642                 die "skip\n";
4643                 # since we cannot reliably hot unplug usb devices
4644                 # we are disabling it
4645                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4646                 vm_deviceunplug($vmid, $conf, $opt);
4647             } elsif ($opt eq 'vcpus') {
4648                 die "skip\n" if !$hotplug_features->{cpu};
4649                 qemu_cpu_hotplug($vmid, $conf, undef);
4650             } elsif ($opt eq 'balloon') {
4651                 # enable balloon device is not hotpluggable
4652                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4653                 # here we reset the ballooning value to memory
4654                 my $balloon = $conf->{memory} || $defaults->{memory};
4655                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4656             } elsif ($fast_plug_option->{$opt}) {
4657                 # do nothing
4658             } elsif ($opt =~ m/^net(\d+)$/) {
4659                 die "skip\n" if !$hotplug_features->{network};
4660                 vm_deviceunplug($vmid, $conf, $opt);
4661             } elsif (is_valid_drivename($opt)) {
4662                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4663                 vm_deviceunplug($vmid, $conf, $opt);
4664                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4665             } elsif ($opt =~ m/^memory$/) {
4666                 die "skip\n" if !$hotplug_features->{memory};
4667                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4668             } elsif ($opt eq 'cpuunits') {
4669                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4670             } elsif ($opt eq 'cpulimit') {
4671                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4672             } else {
4673                 die "skip\n";
4674             }
4675         };
4676         if (my $err = $@) {
4677             &$add_error($opt, $err) if $err ne "skip\n";
4678         } else {
4679             # save new config if hotplug was successful
4680             delete $conf->{$opt};
4681             vmconfig_undelete_pending_option($conf, $opt);
4682             PVE::QemuConfig->write_config($vmid, $conf);
4683             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4684         }
4685     }
4686
4687     my $apply_pending_cloudinit;
4688     $apply_pending_cloudinit = sub {
4689         my ($key, $value) = @_;
4690         $apply_pending_cloudinit = sub {}; # once is enough
4691
4692         my @cloudinit_opts = keys %$confdesc_cloudinit;
4693         foreach my $opt (keys %{$conf->{pending}}) {
4694             next if !grep { $_ eq $opt } @cloudinit_opts;
4695             $conf->{$opt} = delete $conf->{pending}->{$opt};
4696         }
4697
4698         my $new_conf = { %$conf };
4699         $new_conf->{$key} = $value;
4700         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4701     };
4702
4703     foreach my $opt (keys %{$conf->{pending}}) {
4704         next if $selection && !$selection->{$opt};
4705         my $value = $conf->{pending}->{$opt};
4706         eval {
4707             if ($opt eq 'hotplug') {
4708                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4709             } elsif ($opt eq 'tablet') {
4710                 die "skip\n" if !$hotplug_features->{usb};
4711                 if ($value == 1) {
4712                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4713                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4714                         if $arch eq 'aarch64';
4715                 } elsif ($value == 0) {
4716                     vm_deviceunplug($vmid, $conf, 'tablet');
4717                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4718                 }
4719             } elsif ($opt =~ m/^usb\d+$/) {
4720                 die "skip\n";
4721                 # since we cannot reliably hot unplug usb devices
4722                 # we are disabling it
4723                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4724                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4725                 die "skip\n" if !$d;
4726                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4727             } elsif ($opt eq 'vcpus') {
4728                 die "skip\n" if !$hotplug_features->{cpu};
4729                 qemu_cpu_hotplug($vmid, $conf, $value);
4730             } elsif ($opt eq 'balloon') {
4731                 # enable/disable balloning device is not hotpluggable
4732                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4733                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4734                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4735
4736                 # allow manual ballooning if shares is set to zero
4737                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4738                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4739                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4740                 }
4741             } elsif ($opt =~ m/^net(\d+)$/) {
4742                 # some changes can be done without hotplug
4743                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4744                                     $vmid, $opt, $value, $arch, $machine_type);
4745             } elsif (is_valid_drivename($opt)) {
4746                 # some changes can be done without hotplug
4747                 my $drive = parse_drive($opt, $value);
4748                 if (drive_is_cloudinit($drive)) {
4749                     &$apply_pending_cloudinit($opt, $value);
4750                 }
4751                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4752                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4753             } elsif ($opt =~ m/^memory$/) { #dimms
4754                 die "skip\n" if !$hotplug_features->{memory};
4755                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4756             } elsif ($opt eq 'cpuunits') {
4757                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4758             } elsif ($opt eq 'cpulimit') {
4759                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4760                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4761             } else {
4762                 die "skip\n";  # skip non-hot-pluggable options
4763             }
4764         };
4765         if (my $err = $@) {
4766             &$add_error($opt, $err) if $err ne "skip\n";
4767         } else {
4768             # save new config if hotplug was successful
4769             $conf->{$opt} = $value;
4770             delete $conf->{pending}->{$opt};
4771             PVE::QemuConfig->write_config($vmid, $conf);
4772             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4773         }
4774     }
4775 }
4776
4777 sub try_deallocate_drive {
4778     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4779
4780     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4781         my $volid = $drive->{file};
4782         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4783             my $sid = PVE::Storage::parse_volume_id($volid);
4784             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4785
4786             # check if the disk is really unused
4787             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4788                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4789             PVE::Storage::vdisk_free($storecfg, $volid);
4790             return 1;
4791         } else {
4792             # If vm is not owner of this disk remove from config
4793             return 1;
4794         }
4795     }
4796
4797     return undef;
4798 }
4799
4800 sub vmconfig_delete_or_detach_drive {
4801     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4802
4803     my $drive = parse_drive($opt, $conf->{$opt});
4804
4805     my $rpcenv = PVE::RPCEnvironment::get();
4806     my $authuser = $rpcenv->get_user();
4807
4808     if ($force) {
4809         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4810         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4811     } else {
4812         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4813     }
4814 }
4815
4816 sub vmconfig_apply_pending {
4817     my ($vmid, $conf, $storecfg) = @_;
4818
4819     # cold plug
4820
4821     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4822     while (my ($opt, $force) = each %$pending_delete_hash) {
4823         die "internal error" if $opt =~ m/^unused/;
4824         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4825         if (!defined($conf->{$opt})) {
4826             vmconfig_undelete_pending_option($conf, $opt);
4827             PVE::QemuConfig->write_config($vmid, $conf);
4828         } elsif (is_valid_drivename($opt)) {
4829             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4830             vmconfig_undelete_pending_option($conf, $opt);
4831             delete $conf->{$opt};
4832             PVE::QemuConfig->write_config($vmid, $conf);
4833         } else {
4834             vmconfig_undelete_pending_option($conf, $opt);
4835             delete $conf->{$opt};
4836             PVE::QemuConfig->write_config($vmid, $conf);
4837         }
4838     }
4839
4840     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4841
4842     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4843         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4844
4845         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4846             # skip if nothing changed
4847         } elsif (is_valid_drivename($opt)) {
4848             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4849                 if defined($conf->{$opt});
4850             $conf->{$opt} = $conf->{pending}->{$opt};
4851         } else {
4852             $conf->{$opt} = $conf->{pending}->{$opt};
4853         }
4854
4855         delete $conf->{pending}->{$opt};
4856         PVE::QemuConfig->write_config($vmid, $conf);
4857     }
4858 }
4859
4860 my $safe_num_ne = sub {
4861     my ($a, $b) = @_;
4862
4863     return 0 if !defined($a) && !defined($b);
4864     return 1 if !defined($a);
4865     return 1 if !defined($b);
4866
4867     return $a != $b;
4868 };
4869
4870 my $safe_string_ne = sub {
4871     my ($a, $b) = @_;
4872
4873     return 0 if !defined($a) && !defined($b);
4874     return 1 if !defined($a);
4875     return 1 if !defined($b);
4876
4877     return $a ne $b;
4878 };
4879
4880 sub vmconfig_update_net {
4881     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4882
4883     my $newnet = parse_net($value);
4884
4885     if ($conf->{$opt}) {
4886         my $oldnet = parse_net($conf->{$opt});
4887
4888         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4889             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4890             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4891             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4892
4893             # for non online change, we try to hot-unplug
4894             die "skip\n" if !$hotplug;
4895             vm_deviceunplug($vmid, $conf, $opt);
4896         } else {
4897
4898             die "internal error" if $opt !~ m/net(\d+)/;
4899             my $iface = "tap${vmid}i$1";
4900
4901             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4902                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4903                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4904                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4905                 PVE::Network::tap_unplug($iface);
4906                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4907             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4908                 # Rate can be applied on its own but any change above needs to
4909                 # include the rate in tap_plug since OVS resets everything.
4910                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4911             }
4912
4913             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4914                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4915             }
4916
4917             return 1;
4918         }
4919     }
4920
4921     if ($hotplug) {
4922         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4923     } else {
4924         die "skip\n";
4925     }
4926 }
4927
4928 sub vmconfig_update_disk {
4929     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4930
4931     # fixme: do we need force?
4932
4933     my $drive = parse_drive($opt, $value);
4934
4935     if ($conf->{$opt}) {
4936
4937         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4938
4939             my $media = $drive->{media} || 'disk';
4940             my $oldmedia = $old_drive->{media} || 'disk';
4941             die "unable to change media type\n" if $media ne $oldmedia;
4942
4943             if (!drive_is_cdrom($old_drive)) {
4944
4945                 if ($drive->{file} ne $old_drive->{file}) {
4946
4947                     die "skip\n" if !$hotplug;
4948
4949                     # unplug and register as unused
4950                     vm_deviceunplug($vmid, $conf, $opt);
4951                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4952
4953                 } else {
4954                     # update existing disk
4955
4956                     # skip non hotpluggable value
4957                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4958                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4959                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4960                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4961                         die "skip\n";
4962                     }
4963
4964                     # apply throttle
4965                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4966                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4967                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4968                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4969                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4970                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4971                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4972                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4973                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4974                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4975                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4976                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4977                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4978                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4979                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4980                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4981                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4982                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4983
4984                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4985                                                    ($drive->{mbps} || 0)*1024*1024,
4986                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4987                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4988                                                    $drive->{iops} || 0,
4989                                                    $drive->{iops_rd} || 0,
4990                                                    $drive->{iops_wr} || 0,
4991                                                    ($drive->{mbps_max} || 0)*1024*1024,
4992                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4993                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4994                                                    $drive->{iops_max} || 0,
4995                                                    $drive->{iops_rd_max} || 0,
4996                                                    $drive->{iops_wr_max} || 0,
4997                                                    $drive->{bps_max_length} || 1,
4998                                                    $drive->{bps_rd_max_length} || 1,
4999                                                    $drive->{bps_wr_max_length} || 1,
5000                                                    $drive->{iops_max_length} || 1,
5001                                                    $drive->{iops_rd_max_length} || 1,
5002                                                    $drive->{iops_wr_max_length} || 1);
5003
5004                     }
5005
5006                     return 1;
5007                 }
5008
5009             } else { # cdrom
5010
5011                 if ($drive->{file} eq 'none') {
5012                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5013                     if (drive_is_cloudinit($old_drive)) {
5014                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5015                     }
5016                 } else {
5017                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5018                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5019                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5020                 }
5021
5022                 return 1;
5023             }
5024         }
5025     }
5026
5027     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5028     # hotplug new disks
5029     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5030     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5031 }
5032
5033 sub vm_start {
5034     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5035         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5036
5037     PVE::QemuConfig->lock_config($vmid, sub {
5038         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5039
5040         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5041
5042         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5043
5044         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5045
5046         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5047             vmconfig_apply_pending($vmid, $conf, $storecfg);
5048             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5049         }
5050
5051         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5052
5053         my $defaults = load_defaults();
5054
5055         # set environment variable useful inside network script
5056         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5057
5058         my $local_volumes = {};
5059
5060         if ($targetstorage) {
5061             foreach_drive($conf, sub {
5062                 my ($ds, $drive) = @_;
5063
5064                 return if drive_is_cdrom($drive);
5065
5066                 my $volid = $drive->{file};
5067
5068                 return if !$volid;
5069
5070                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5071
5072                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5073                 return if $scfg->{shared};
5074                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5075             });
5076
5077             my $format = undef;
5078
5079             foreach my $opt (sort keys %$local_volumes) {
5080
5081                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5082                 my $drive = parse_drive($opt, $conf->{$opt});
5083
5084                 #if remote storage is specified, use default format
5085                 if ($targetstorage && $targetstorage ne "1") {
5086                     $storeid = $targetstorage;
5087                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5088                     $format = $defFormat;
5089                 } else {
5090                     #else we use same format than original
5091                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5092                     $format = qemu_img_format($scfg, $volid);
5093                 }
5094
5095                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5096                 my $newdrive = $drive;
5097                 $newdrive->{format} = $format;
5098                 $newdrive->{file} = $newvolid;
5099                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5100                 $local_volumes->{$opt} = $drivestr;
5101                 #pass drive to conf for command line
5102                 $conf->{$opt} = $drivestr;
5103             }
5104         }
5105
5106         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5107
5108         my $migrate_port = 0;
5109         my $migrate_uri;
5110         if ($statefile) {
5111             if ($statefile eq 'tcp') {
5112                 my $localip = "localhost";
5113                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5114                 my $nodename = PVE::INotify::nodename();
5115
5116                 if (!defined($migration_type)) {
5117                     if (defined($datacenterconf->{migration}->{type})) {
5118                         $migration_type = $datacenterconf->{migration}->{type};
5119                     } else {
5120                         $migration_type = 'secure';
5121                     }
5122                 }
5123
5124                 if ($migration_type eq 'insecure') {
5125                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5126                     if ($migrate_network_addr) {
5127                         $localip = $migrate_network_addr;
5128                     } else {
5129                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5130                     }
5131
5132                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5133                 }
5134
5135                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5136                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5137                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5138                 push @$cmd, '-incoming', $migrate_uri;
5139                 push @$cmd, '-S';
5140
5141             } elsif ($statefile eq 'unix') {
5142                 # should be default for secure migrations as a ssh TCP forward
5143                 # tunnel is not deterministic reliable ready and fails regurarly
5144                 # to set up in time, so use UNIX socket forwards
5145                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5146                 unlink $socket_addr;
5147
5148                 $migrate_uri = "unix:$socket_addr";
5149
5150                 push @$cmd, '-incoming', $migrate_uri;
5151                 push @$cmd, '-S';
5152
5153             } else {
5154                 push @$cmd, '-loadstate', $statefile;
5155             }
5156         } elsif ($paused) {
5157             push @$cmd, '-S';
5158         }
5159
5160         # host pci devices
5161         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5162           my $d = parse_hostpci($conf->{"hostpci$i"});
5163           next if !$d;
5164           my $pcidevices = $d->{pciid};
5165           foreach my $pcidevice (@$pcidevices) {
5166                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5167
5168                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5169                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5170                 die "no pci device info for device '$pciid'\n" if !$info;
5171
5172                 if ($d->{mdev}) {
5173                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5174                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5175                 } else {
5176                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5177                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5178                     die "can't reset pci device '$pciid'\n"
5179                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5180                 }
5181           }
5182         }
5183
5184         PVE::Storage::activate_volumes($storecfg, $vollist);
5185
5186         if (!check_running($vmid, 1)) {
5187             eval {
5188                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5189                     outfunc => sub {}, errfunc => sub {});
5190             };
5191         }
5192
5193         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5194                                                   : $defaults->{cpuunits};
5195
5196         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5197         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5198
5199         my %properties = (
5200             Slice => 'qemu.slice',
5201             KillMode => 'none',
5202             CPUShares => $cpuunits
5203         );
5204
5205         if (my $cpulimit = $conf->{cpulimit}) {
5206             $properties{CPUQuota} = int($cpulimit * 100);
5207         }
5208         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5209
5210         my $run_qemu = sub {
5211             PVE::Tools::run_fork sub {
5212                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5213                 run_command($cmd, %run_params);
5214             };
5215         };
5216
5217         if ($conf->{hugepages}) {
5218
5219             my $code = sub {
5220                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5221                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5222
5223                 PVE::QemuServer::Memory::hugepages_mount();
5224                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5225
5226                 eval { $run_qemu->() };
5227                 if (my $err = $@) {
5228                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5229                     die $err;
5230                 }
5231
5232                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5233             };
5234             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5235
5236         } else {
5237             eval { $run_qemu->() };
5238         }
5239
5240         if (my $err = $@) {
5241             # deactivate volumes if start fails
5242             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5243             die "start failed: $err";
5244         }
5245
5246         print "migration listens on $migrate_uri\n" if $migrate_uri;
5247
5248         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5249             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5250             warn $@ if $@;
5251         }
5252
5253         #start nbd server for storage migration
5254         if ($targetstorage) {
5255             my $nodename = PVE::INotify::nodename();
5256             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5257             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5258             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5259             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5260
5261             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5262
5263             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5264
5265             foreach my $opt (sort keys %$local_volumes) {
5266                 my $volid = $local_volumes->{$opt};
5267                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5268                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5269                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5270             }
5271         }
5272
5273         if ($migratedfrom) {
5274             eval {
5275                 set_migration_caps($vmid);
5276             };
5277             warn $@ if $@;
5278
5279             if ($spice_port) {
5280                 print "spice listens on port $spice_port\n";
5281                 if ($spice_ticket) {
5282                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5283                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5284                 }
5285             }
5286
5287         } else {
5288             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5289                 if !$statefile && $conf->{balloon};
5290
5291             foreach my $opt (keys %$conf) {
5292                 next if $opt !~  m/^net\d+$/;
5293                 my $nicconf = parse_net($conf->{$opt});
5294                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5295             }
5296         }
5297
5298         vm_mon_cmd_nocheck($vmid, 'qom-set',
5299                     path => "machine/peripheral/balloon0",
5300                     property => "guest-stats-polling-interval",
5301                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5302
5303     });
5304 }
5305
5306 sub vm_mon_cmd {
5307     my ($vmid, $execute, %params) = @_;
5308
5309     my $cmd = { execute => $execute, arguments => \%params };
5310     vm_qmp_command($vmid, $cmd);
5311 }
5312
5313 sub vm_mon_cmd_nocheck {
5314     my ($vmid, $execute, %params) = @_;
5315
5316     my $cmd = { execute => $execute, arguments => \%params };
5317     vm_qmp_command($vmid, $cmd, 1);
5318 }
5319
5320 sub vm_qmp_command {
5321     my ($vmid, $cmd, $nocheck) = @_;
5322
5323     my $res;
5324
5325     my $timeout;
5326     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5327         $timeout = $cmd->{arguments}->{timeout};
5328         delete $cmd->{arguments}->{timeout};
5329     }
5330
5331     eval {
5332         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5333         my $sname = qmp_socket($vmid);
5334         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5335             my $qmpclient = PVE::QMPClient->new();
5336
5337             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5338         } else {
5339             die "unable to open monitor socket\n";
5340         }
5341     };
5342     if (my $err = $@) {
5343         syslog("err", "VM $vmid qmp command failed - $err");
5344         die $err;
5345     }
5346
5347     return $res;
5348 }
5349
5350 sub vm_human_monitor_command {
5351     my ($vmid, $cmdline) = @_;
5352
5353     my $res;
5354
5355     my $cmd = {
5356         execute => 'human-monitor-command',
5357         arguments => { 'command-line' => $cmdline},
5358     };
5359
5360     return vm_qmp_command($vmid, $cmd);
5361 }
5362
5363 sub vm_commandline {
5364     my ($storecfg, $vmid) = @_;
5365
5366     my $conf = PVE::QemuConfig->load_config($vmid);
5367
5368     my $defaults = load_defaults();
5369
5370     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5371
5372     return PVE::Tools::cmd2string($cmd);
5373 }
5374
5375 sub vm_reset {
5376     my ($vmid, $skiplock) = @_;
5377
5378     PVE::QemuConfig->lock_config($vmid, sub {
5379
5380         my $conf = PVE::QemuConfig->load_config($vmid);
5381
5382         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5383
5384         vm_mon_cmd($vmid, "system_reset");
5385     });
5386 }
5387
5388 sub get_vm_volumes {
5389     my ($conf) = @_;
5390
5391     my $vollist = [];
5392     foreach_volid($conf, sub {
5393         my ($volid, $attr) = @_;
5394
5395         return if $volid =~ m|^/|;
5396
5397         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5398         return if !$sid;
5399
5400         push @$vollist, $volid;
5401     });
5402
5403     return $vollist;
5404 }
5405
5406 sub vm_stop_cleanup {
5407     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5408
5409     eval {
5410
5411         if (!$keepActive) {
5412             my $vollist = get_vm_volumes($conf);
5413             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5414         }
5415
5416         foreach my $ext (qw(mon qmp pid vnc qga)) {
5417             unlink "/var/run/qemu-server/${vmid}.$ext";
5418         }
5419
5420         foreach my $key (keys %$conf) {
5421             next if $key !~ m/^hostpci(\d+)$/;
5422             my $hostpciindex = $1;
5423             my $d = parse_hostpci($conf->{$key});
5424             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5425
5426             foreach my $pci (@{$d->{pciid}}) {
5427                 my $pciid = $pci->{id} . "." . $pci->{function};
5428                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5429             }
5430         }
5431
5432         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5433     };
5434     warn $@ if $@; # avoid errors - just warn
5435 }
5436
5437 # Note: use $nockeck to skip tests if VM configuration file exists.
5438 # We need that when migration VMs to other nodes (files already moved)
5439 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5440 sub vm_stop {
5441     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5442
5443     $force = 1 if !defined($force) && !$shutdown;
5444
5445     if ($migratedfrom){
5446         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5447         kill 15, $pid if $pid;
5448         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5449         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5450         return;
5451     }
5452
5453     PVE::QemuConfig->lock_config($vmid, sub {
5454
5455         my $pid = check_running($vmid, $nocheck);
5456         return if !$pid;
5457
5458         my $conf;
5459         if (!$nocheck) {
5460             $conf = PVE::QemuConfig->load_config($vmid);
5461             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5462             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5463                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5464                 $timeout = $opts->{down} if $opts->{down};
5465             }
5466         }
5467
5468         $timeout = 60 if !defined($timeout);
5469
5470         eval {
5471             if ($shutdown) {
5472                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5473                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5474                 } else {
5475                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5476                 }
5477             } else {
5478                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5479             }
5480         };
5481         my $err = $@;
5482
5483         if (!$err) {
5484             my $count = 0;
5485             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5486                 $count++;
5487                 sleep 1;
5488             }
5489
5490             if ($count >= $timeout) {
5491                 if ($force) {
5492                     warn "VM still running - terminating now with SIGTERM\n";
5493                     kill 15, $pid;
5494                 } else {
5495                     die "VM quit/powerdown failed - got timeout\n";
5496                 }
5497             } else {
5498                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5499                 return;
5500             }
5501         } else {
5502             if ($force) {
5503                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5504                 kill 15, $pid;
5505             } else {
5506                 die "VM quit/powerdown failed\n";
5507             }
5508         }
5509
5510         # wait again
5511         $timeout = 10;
5512
5513         my $count = 0;
5514         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5515             $count++;
5516             sleep 1;
5517         }
5518
5519         if ($count >= $timeout) {
5520             warn "VM still running - terminating now with SIGKILL\n";
5521             kill 9, $pid;
5522             sleep 1;
5523         }
5524
5525         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5526    });
5527 }
5528
5529 sub vm_suspend {
5530     my ($vmid, $skiplock) = @_;
5531
5532     PVE::QemuConfig->lock_config($vmid, sub {
5533
5534         my $conf = PVE::QemuConfig->load_config($vmid);
5535
5536         PVE::QemuConfig->check_lock($conf)
5537             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5538
5539         vm_mon_cmd($vmid, "stop");
5540     });
5541 }
5542
5543 sub vm_resume {
5544     my ($vmid, $skiplock, $nocheck) = @_;
5545
5546     PVE::QemuConfig->lock_config($vmid, sub {
5547
5548         my $res = vm_mon_cmd($vmid, 'query-status');
5549         my $resume_cmd = 'cont';
5550
5551         if ($res->{status} && $res->{status} eq 'suspended') {
5552             $resume_cmd = 'system_wakeup';
5553         }
5554
5555         if (!$nocheck) {
5556
5557             my $conf = PVE::QemuConfig->load_config($vmid);
5558
5559             PVE::QemuConfig->check_lock($conf)
5560                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5561
5562             vm_mon_cmd($vmid, $resume_cmd);
5563
5564         } else {
5565             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5566         }
5567     });
5568 }
5569
5570 sub vm_sendkey {
5571     my ($vmid, $skiplock, $key) = @_;
5572
5573     PVE::QemuConfig->lock_config($vmid, sub {
5574
5575         my $conf = PVE::QemuConfig->load_config($vmid);
5576
5577         # there is no qmp command, so we use the human monitor command
5578         vm_human_monitor_command($vmid, "sendkey $key");
5579     });
5580 }
5581
5582 sub vm_destroy {
5583     my ($storecfg, $vmid, $skiplock) = @_;
5584
5585     PVE::QemuConfig->lock_config($vmid, sub {
5586
5587         my $conf = PVE::QemuConfig->load_config($vmid);
5588
5589         if (!check_running($vmid)) {
5590             destroy_vm($storecfg, $vmid, undef, $skiplock);
5591         } else {
5592             die "VM $vmid is running - destroy failed\n";
5593         }
5594     });
5595 }
5596
5597 # vzdump restore implementaion
5598
5599 sub tar_archive_read_firstfile {
5600     my $archive = shift;
5601
5602     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5603
5604     # try to detect archive type first
5605     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5606         die "unable to open file '$archive'\n";
5607     my $firstfile = <$fh>;
5608     kill 15, $pid;
5609     close $fh;
5610
5611     die "ERROR: archive contaions no data\n" if !$firstfile;
5612     chomp $firstfile;
5613
5614     return $firstfile;
5615 }
5616
5617 sub tar_restore_cleanup {
5618     my ($storecfg, $statfile) = @_;
5619
5620     print STDERR "starting cleanup\n";
5621
5622     if (my $fd = IO::File->new($statfile, "r")) {
5623         while (defined(my $line = <$fd>)) {
5624             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5625                 my $volid = $2;
5626                 eval {
5627                     if ($volid =~ m|^/|) {
5628                         unlink $volid || die 'unlink failed\n';
5629                     } else {
5630                         PVE::Storage::vdisk_free($storecfg, $volid);
5631                     }
5632                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5633                 };
5634                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5635             } else {
5636                 print STDERR "unable to parse line in statfile - $line";
5637             }
5638         }
5639         $fd->close();
5640     }
5641 }
5642
5643 sub restore_archive {
5644     my ($archive, $vmid, $user, $opts) = @_;
5645
5646     my $format = $opts->{format};
5647     my $comp;
5648
5649     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5650         $format = 'tar' if !$format;
5651         $comp = 'gzip';
5652     } elsif ($archive =~ m/\.tar$/) {
5653         $format = 'tar' if !$format;
5654     } elsif ($archive =~ m/.tar.lzo$/) {
5655         $format = 'tar' if !$format;
5656         $comp = 'lzop';
5657     } elsif ($archive =~ m/\.vma$/) {
5658         $format = 'vma' if !$format;
5659     } elsif ($archive =~ m/\.vma\.gz$/) {
5660         $format = 'vma' if !$format;
5661         $comp = 'gzip';
5662     } elsif ($archive =~ m/\.vma\.lzo$/) {
5663         $format = 'vma' if !$format;
5664         $comp = 'lzop';
5665     } else {
5666         $format = 'vma' if !$format; # default
5667     }
5668
5669     # try to detect archive format
5670     if ($format eq 'tar') {
5671         return restore_tar_archive($archive, $vmid, $user, $opts);
5672     } else {
5673         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5674     }
5675 }
5676
5677 sub restore_update_config_line {
5678     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5679
5680     return if $line =~ m/^\#qmdump\#/;
5681     return if $line =~ m/^\#vzdump\#/;
5682     return if $line =~ m/^lock:/;
5683     return if $line =~ m/^unused\d+:/;
5684     return if $line =~ m/^parent:/;
5685     return if $line =~ m/^template:/; # restored VM is never a template
5686
5687     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5688     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5689         # try to convert old 1.X settings
5690         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5691         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5692             my ($model, $macaddr) = split(/\=/, $devconfig);
5693             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5694             my $net = {
5695                 model => $model,
5696                 bridge => "vmbr$ind",
5697                 macaddr => $macaddr,
5698             };
5699             my $netstr = print_net($net);
5700
5701             print $outfd "net$cookie->{netcount}: $netstr\n";
5702             $cookie->{netcount}++;
5703         }
5704     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5705         my ($id, $netstr) = ($1, $2);
5706         my $net = parse_net($netstr);
5707         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5708         $netstr = print_net($net);
5709         print $outfd "$id: $netstr\n";
5710     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5711         my $virtdev = $1;
5712         my $value = $3;
5713         my $di = parse_drive($virtdev, $value);
5714         if (defined($di->{backup}) && !$di->{backup}) {
5715             print $outfd "#$line";
5716         } elsif ($map->{$virtdev}) {
5717             delete $di->{format}; # format can change on restore
5718             $di->{file} = $map->{$virtdev};
5719             $value = print_drive($vmid, $di);
5720             print $outfd "$virtdev: $value\n";
5721         } else {
5722             print $outfd $line;
5723         }
5724     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5725         my $vmgenid = $1;
5726         if ($vmgenid ne '0') {
5727             # always generate a new vmgenid if there was a valid one setup
5728             $vmgenid = generate_uuid();
5729         }
5730         print $outfd "vmgenid: $vmgenid\n";
5731     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5732         my ($uuid, $uuid_str);
5733         UUID::generate($uuid);
5734         UUID::unparse($uuid, $uuid_str);
5735         my $smbios1 = parse_smbios1($2);
5736         $smbios1->{uuid} = $uuid_str;
5737         print $outfd $1.print_smbios1($smbios1)."\n";
5738     } else {
5739         print $outfd $line;
5740     }
5741 }
5742
5743 sub scan_volids {
5744     my ($cfg, $vmid) = @_;
5745
5746     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5747
5748     my $volid_hash = {};
5749     foreach my $storeid (keys %$info) {
5750         foreach my $item (@{$info->{$storeid}}) {
5751             next if !($item->{volid} && $item->{size});
5752             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5753             $volid_hash->{$item->{volid}} = $item;
5754         }
5755     }
5756
5757     return $volid_hash;
5758 }
5759
5760 sub is_volume_in_use {
5761     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5762
5763     my $path = PVE::Storage::path($storecfg, $volid);
5764
5765     my $scan_config = sub {
5766         my ($cref, $snapname) = @_;
5767
5768         foreach my $key (keys %$cref) {
5769             my $value = $cref->{$key};
5770             if (is_valid_drivename($key)) {
5771                 next if $skip_drive && $key eq $skip_drive;
5772                 my $drive = parse_drive($key, $value);
5773                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5774                 return 1 if $volid eq $drive->{file};
5775                 if ($drive->{file} =~ m!^/!) {
5776                     return 1 if $drive->{file} eq $path;
5777                 } else {
5778                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5779                     next if !$storeid;
5780                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5781                     next if !$scfg;
5782                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5783                 }
5784             }
5785         }
5786
5787         return 0;
5788     };
5789
5790     return 1 if &$scan_config($conf);
5791
5792     undef $skip_drive;
5793
5794     foreach my $snapname (keys %{$conf->{snapshots}}) {
5795         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5796     }
5797
5798     return 0;
5799 }
5800
5801 sub update_disksize {
5802     my ($vmid, $conf, $volid_hash) = @_;
5803
5804     my $changes;
5805     my $prefix = "VM $vmid:";
5806
5807     # used and unused disks
5808     my $referenced = {};
5809
5810     # Note: it is allowed to define multiple storages with same path (alias), so
5811     # we need to check both 'volid' and real 'path' (two different volid can point
5812     # to the same path).
5813
5814     my $referencedpath = {};
5815
5816     # update size info
5817     foreach my $opt (keys %$conf) {
5818         if (is_valid_drivename($opt)) {
5819             my $drive = parse_drive($opt, $conf->{$opt});
5820             my $volid = $drive->{file};
5821             next if !$volid;
5822
5823             $referenced->{$volid} = 1;
5824             if ($volid_hash->{$volid} &&
5825                 (my $path = $volid_hash->{$volid}->{path})) {
5826                 $referencedpath->{$path} = 1;
5827             }
5828
5829             next if drive_is_cdrom($drive);
5830             next if !$volid_hash->{$volid};
5831
5832             $drive->{size} = $volid_hash->{$volid}->{size};
5833             my $new = print_drive($vmid, $drive);
5834             if ($new ne $conf->{$opt}) {
5835                 $changes = 1;
5836                 $conf->{$opt} = $new;
5837                 print "$prefix update disk '$opt' information.\n";
5838             }
5839         }
5840     }
5841
5842     # remove 'unusedX' entry if volume is used
5843     foreach my $opt (keys %$conf) {
5844         next if $opt !~ m/^unused\d+$/;
5845         my $volid = $conf->{$opt};
5846         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5847         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5848             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5849             $changes = 1;
5850             delete $conf->{$opt};
5851         }
5852
5853         $referenced->{$volid} = 1;
5854         $referencedpath->{$path} = 1 if $path;
5855     }
5856
5857     foreach my $volid (sort keys %$volid_hash) {
5858         next if $volid =~ m/vm-$vmid-state-/;
5859         next if $referenced->{$volid};
5860         my $path = $volid_hash->{$volid}->{path};
5861         next if !$path; # just to be sure
5862         next if $referencedpath->{$path};
5863         $changes = 1;
5864         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5865         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5866         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5867     }
5868
5869     return $changes;
5870 }
5871
5872 sub rescan {
5873     my ($vmid, $nolock, $dryrun) = @_;
5874
5875     my $cfg = PVE::Storage::config();
5876
5877     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5878     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5879     foreach my $stor (keys %{$cfg->{ids}}) {
5880         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5881     }
5882
5883     print "rescan volumes...\n";
5884     my $volid_hash = scan_volids($cfg, $vmid);
5885
5886     my $updatefn =  sub {
5887         my ($vmid) = @_;
5888
5889         my $conf = PVE::QemuConfig->load_config($vmid);
5890
5891         PVE::QemuConfig->check_lock($conf);
5892
5893         my $vm_volids = {};
5894         foreach my $volid (keys %$volid_hash) {
5895             my $info = $volid_hash->{$volid};
5896             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5897         }
5898
5899         my $changes = update_disksize($vmid, $conf, $vm_volids);
5900
5901         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5902     };
5903
5904     if (defined($vmid)) {
5905         if ($nolock) {
5906             &$updatefn($vmid);
5907         } else {
5908             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5909         }
5910     } else {
5911         my $vmlist = config_list();
5912         foreach my $vmid (keys %$vmlist) {
5913             if ($nolock) {
5914                 &$updatefn($vmid);
5915             } else {
5916                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5917             }
5918         }
5919     }
5920 }
5921
5922 sub restore_vma_archive {
5923     my ($archive, $vmid, $user, $opts, $comp) = @_;
5924
5925     my $readfrom = $archive;
5926
5927     my $cfg = PVE::Storage::config();
5928     my $commands = [];
5929     my $bwlimit = $opts->{bwlimit};
5930
5931     my $dbg_cmdstring = '';
5932     my $add_pipe = sub {
5933         my ($cmd) = @_;
5934         push @$commands, $cmd;
5935         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5936         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5937         $readfrom = '-';
5938     };
5939
5940     my $input = undef;
5941     if ($archive eq '-') {
5942         $input = '<&STDIN';
5943     } else {
5944         # If we use a backup from a PVE defined storage we also consider that
5945         # storage's rate limit:
5946         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5947         if (defined($volid)) {
5948             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5949             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5950             if ($readlimit) {
5951                 print STDERR "applying read rate limit: $readlimit\n";
5952                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5953                 $add_pipe->($cstream);
5954             }
5955         }
5956     }
5957
5958     if ($comp) {
5959         my $cmd;
5960         if ($comp eq 'gzip') {
5961             $cmd = ['zcat', $readfrom];
5962         } elsif ($comp eq 'lzop') {
5963             $cmd = ['lzop', '-d', '-c', $readfrom];
5964         } else {
5965             die "unknown compression method '$comp'\n";
5966         }
5967         $add_pipe->($cmd);
5968     }
5969
5970     my $tmpdir = "/var/tmp/vzdumptmp$$";
5971     rmtree $tmpdir;
5972
5973     # disable interrupts (always do cleanups)
5974     local $SIG{INT} =
5975         local $SIG{TERM} =
5976         local $SIG{QUIT} =
5977         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5978
5979     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5980     POSIX::mkfifo($mapfifo, 0600);
5981     my $fifofh;
5982
5983     my $openfifo = sub {
5984         open($fifofh, '>', $mapfifo) || die $!;
5985     };
5986
5987     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5988
5989     my $oldtimeout;
5990     my $timeout = 5;
5991
5992     my $devinfo = {};
5993
5994     my $rpcenv = PVE::RPCEnvironment::get();
5995
5996     my $conffile = PVE::QemuConfig->config_file($vmid);
5997     my $tmpfn = "$conffile.$$.tmp";
5998
5999     # Note: $oldconf is undef if VM does not exists
6000     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6001     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6002
6003     my %storage_limits;
6004
6005     my $print_devmap = sub {
6006         my $virtdev_hash = {};
6007
6008         my $cfgfn = "$tmpdir/qemu-server.conf";
6009
6010         # we can read the config - that is already extracted
6011         my $fh = IO::File->new($cfgfn, "r") ||
6012             "unable to read qemu-server.conf - $!\n";
6013
6014         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6015         if (-f $fwcfgfn) {
6016             my $pve_firewall_dir = '/etc/pve/firewall';
6017             mkdir $pve_firewall_dir; # make sure the dir exists
6018             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6019         }
6020
6021         while (defined(my $line = <$fh>)) {
6022             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6023                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6024                 die "archive does not contain data for drive '$virtdev'\n"
6025                     if !$devinfo->{$devname};
6026                 if (defined($opts->{storage})) {
6027                     $storeid = $opts->{storage} || 'local';
6028                 } elsif (!$storeid) {
6029                     $storeid = 'local';
6030                 }
6031                 $format = 'raw' if !$format;
6032                 $devinfo->{$devname}->{devname} = $devname;
6033                 $devinfo->{$devname}->{virtdev} = $virtdev;
6034                 $devinfo->{$devname}->{format} = $format;
6035                 $devinfo->{$devname}->{storeid} = $storeid;
6036
6037                 # check permission on storage
6038                 my $pool = $opts->{pool}; # todo: do we need that?
6039                 if ($user ne 'root@pam') {
6040                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6041                 }
6042
6043                 $storage_limits{$storeid} = $bwlimit;
6044
6045                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6046             }
6047         }
6048
6049         foreach my $key (keys %storage_limits) {
6050             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6051             next if !$limit;
6052             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6053             $storage_limits{$key} = $limit * 1024;
6054         }
6055
6056         foreach my $devname (keys %$devinfo) {
6057             die "found no device mapping information for device '$devname'\n"
6058                 if !$devinfo->{$devname}->{virtdev};
6059         }
6060
6061         # create empty/temp config
6062         if ($oldconf) {
6063             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6064             foreach_drive($oldconf, sub {
6065                 my ($ds, $drive) = @_;
6066
6067                 return if drive_is_cdrom($drive);
6068
6069                 my $volid = $drive->{file};
6070
6071                 return if !$volid || $volid =~ m|^/|;
6072
6073                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6074                 return if !$path || !$owner || ($owner != $vmid);
6075
6076                 # Note: only delete disk we want to restore
6077                 # other volumes will become unused
6078                 if ($virtdev_hash->{$ds}) {
6079                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6080                     if (my $err = $@) {
6081                         warn $err;
6082                     }
6083                 }
6084             });
6085
6086             # delete vmstate files
6087             # since after the restore we have no snapshots anymore
6088             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6089                 my $snap = $oldconf->{snapshots}->{$snapname};
6090                 if ($snap->{vmstate}) {
6091                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6092                     if (my $err = $@) {
6093                         warn $err;
6094                     }
6095                 }
6096             }
6097         }
6098
6099         my $map = {};
6100         foreach my $virtdev (sort keys %$virtdev_hash) {
6101             my $d = $virtdev_hash->{$virtdev};
6102             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6103             my $storeid = $d->{storeid};
6104             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6105
6106             my $map_opts = '';
6107             if (my $limit = $storage_limits{$storeid}) {
6108                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6109             }
6110
6111             # test if requested format is supported
6112             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6113             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6114             $d->{format} = $defFormat if !$supported;
6115
6116             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6117                                                   $d->{format}, undef, $alloc_size);
6118             print STDERR "new volume ID is '$volid'\n";
6119             $d->{volid} = $volid;
6120             my $path = PVE::Storage::path($cfg, $volid);
6121
6122             PVE::Storage::activate_volumes($cfg,[$volid]);
6123
6124             my $write_zeros = 1;
6125             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6126                 $write_zeros = 0;
6127             }
6128
6129             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6130
6131             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6132             $map->{$virtdev} = $volid;
6133         }
6134
6135         $fh->seek(0, 0) || die "seek failed - $!\n";
6136
6137         my $outfd = new IO::File ($tmpfn, "w") ||
6138             die "unable to write config for VM $vmid\n";
6139
6140         my $cookie = { netcount => 0 };
6141         while (defined(my $line = <$fh>)) {
6142             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6143         }
6144
6145         $fh->close();
6146         $outfd->close();
6147     };
6148
6149     eval {
6150         # enable interrupts
6151         local $SIG{INT} =
6152             local $SIG{TERM} =
6153             local $SIG{QUIT} =
6154             local $SIG{HUP} =
6155             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6156         local $SIG{ALRM} = sub { die "got timeout\n"; };
6157
6158         $oldtimeout = alarm($timeout);
6159
6160         my $parser = sub {
6161             my $line = shift;
6162
6163             print "$line\n";
6164
6165             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6166                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6167                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6168             } elsif ($line =~ m/^CTIME: /) {
6169                 # we correctly received the vma config, so we can disable
6170                 # the timeout now for disk allocation (set to 10 minutes, so
6171                 # that we always timeout if something goes wrong)
6172                 alarm(600);
6173                 &$print_devmap();
6174                 print $fifofh "done\n";
6175                 my $tmp = $oldtimeout || 0;
6176                 $oldtimeout = undef;
6177                 alarm($tmp);
6178                 close($fifofh);
6179             }
6180         };
6181
6182         print "restore vma archive: $dbg_cmdstring\n";
6183         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6184     };
6185     my $err = $@;
6186
6187     alarm($oldtimeout) if $oldtimeout;
6188
6189     my $vollist = [];
6190     foreach my $devname (keys %$devinfo) {
6191         my $volid = $devinfo->{$devname}->{volid};
6192         push @$vollist, $volid if $volid;
6193     }
6194
6195     PVE::Storage::deactivate_volumes($cfg, $vollist);
6196
6197     unlink $mapfifo;
6198
6199     if ($err) {
6200         rmtree $tmpdir;
6201         unlink $tmpfn;
6202
6203         foreach my $devname (keys %$devinfo) {
6204             my $volid = $devinfo->{$devname}->{volid};
6205             next if !$volid;
6206             eval {
6207                 if ($volid =~ m|^/|) {
6208                     unlink $volid || die 'unlink failed\n';
6209                 } else {
6210                     PVE::Storage::vdisk_free($cfg, $volid);
6211                 }
6212                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6213             };
6214             print STDERR "unable to cleanup '$volid' - $@" if $@;
6215         }
6216         die $err;
6217     }
6218
6219     rmtree $tmpdir;
6220
6221     rename($tmpfn, $conffile) ||
6222         die "unable to commit configuration file '$conffile'\n";
6223
6224     PVE::Cluster::cfs_update(); # make sure we read new file
6225
6226     eval { rescan($vmid, 1); };
6227     warn $@ if $@;
6228 }
6229
6230 sub restore_tar_archive {
6231     my ($archive, $vmid, $user, $opts) = @_;
6232
6233     if ($archive ne '-') {
6234         my $firstfile = tar_archive_read_firstfile($archive);
6235         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6236             if $firstfile ne 'qemu-server.conf';
6237     }
6238
6239     my $storecfg = PVE::Storage::config();
6240
6241     # destroy existing data - keep empty config
6242     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6243     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6244
6245     my $tocmd = "/usr/lib/qemu-server/qmextract";
6246
6247     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6248     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6249     $tocmd .= ' --prealloc' if $opts->{prealloc};
6250     $tocmd .= ' --info' if $opts->{info};
6251
6252     # tar option "xf" does not autodetect compression when read from STDIN,
6253     # so we pipe to zcat
6254     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6255         PVE::Tools::shellquote("--to-command=$tocmd");
6256
6257     my $tmpdir = "/var/tmp/vzdumptmp$$";
6258     mkpath $tmpdir;
6259
6260     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6261     local $ENV{VZDUMP_VMID} = $vmid;
6262     local $ENV{VZDUMP_USER} = $user;
6263
6264     my $conffile = PVE::QemuConfig->config_file($vmid);
6265     my $tmpfn = "$conffile.$$.tmp";
6266
6267     # disable interrupts (always do cleanups)
6268     local $SIG{INT} =
6269         local $SIG{TERM} =
6270         local $SIG{QUIT} =
6271         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6272
6273     eval {
6274         # enable interrupts
6275         local $SIG{INT} =
6276             local $SIG{TERM} =
6277             local $SIG{QUIT} =
6278             local $SIG{HUP} =
6279             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6280
6281         if ($archive eq '-') {
6282             print "extracting archive from STDIN\n";
6283             run_command($cmd, input => "<&STDIN");
6284         } else {
6285             print "extracting archive '$archive'\n";
6286             run_command($cmd);
6287         }
6288
6289         return if $opts->{info};
6290
6291         # read new mapping
6292         my $map = {};
6293         my $statfile = "$tmpdir/qmrestore.stat";
6294         if (my $fd = IO::File->new($statfile, "r")) {
6295             while (defined (my $line = <$fd>)) {
6296                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6297                     $map->{$1} = $2 if $1;
6298                 } else {
6299                     print STDERR "unable to parse line in statfile - $line\n";
6300                 }
6301             }
6302             $fd->close();
6303         }
6304
6305         my $confsrc = "$tmpdir/qemu-server.conf";
6306
6307         my $srcfd = new IO::File($confsrc, "r") ||
6308             die "unable to open file '$confsrc'\n";
6309
6310         my $outfd = new IO::File ($tmpfn, "w") ||
6311             die "unable to write config for VM $vmid\n";
6312
6313         my $cookie = { netcount => 0 };
6314         while (defined (my $line = <$srcfd>)) {
6315             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6316         }
6317
6318         $srcfd->close();
6319         $outfd->close();
6320     };
6321     my $err = $@;
6322
6323     if ($err) {
6324
6325         unlink $tmpfn;
6326
6327         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6328
6329         die $err;
6330     }
6331
6332     rmtree $tmpdir;
6333
6334     rename $tmpfn, $conffile ||
6335         die "unable to commit configuration file '$conffile'\n";
6336
6337     PVE::Cluster::cfs_update(); # make sure we read new file
6338
6339     eval { rescan($vmid, 1); };
6340     warn $@ if $@;
6341 };
6342
6343 sub foreach_storage_used_by_vm {
6344     my ($conf, $func) = @_;
6345
6346     my $sidhash = {};
6347
6348     foreach_drive($conf, sub {
6349         my ($ds, $drive) = @_;
6350         return if drive_is_cdrom($drive);
6351
6352         my $volid = $drive->{file};
6353
6354         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6355         $sidhash->{$sid} = $sid if $sid;
6356     });
6357
6358     foreach my $sid (sort keys %$sidhash) {
6359         &$func($sid);
6360     }
6361 }
6362
6363 sub do_snapshots_with_qemu {
6364     my ($storecfg, $volid) = @_;
6365
6366     my $storage_name = PVE::Storage::parse_volume_id($volid);
6367
6368     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6369         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6370         return 1;
6371     }
6372
6373     if ($volid =~ m/\.(qcow2|qed)$/){
6374         return 1;
6375     }
6376
6377     return undef;
6378 }
6379
6380 sub qga_check_running {
6381     my ($vmid, $nowarn) = @_;
6382
6383     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6384     if ($@) {
6385         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6386         return 0;
6387     }
6388     return 1;
6389 }
6390
6391 sub template_create {
6392     my ($vmid, $conf, $disk) = @_;
6393
6394     my $storecfg = PVE::Storage::config();
6395
6396     foreach_drive($conf, sub {
6397         my ($ds, $drive) = @_;
6398
6399         return if drive_is_cdrom($drive);
6400         return if $disk && $ds ne $disk;
6401
6402         my $volid = $drive->{file};
6403         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6404
6405         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6406         $drive->{file} = $voliddst;
6407         $conf->{$ds} = print_drive($vmid, $drive);
6408         PVE::QemuConfig->write_config($vmid, $conf);
6409     });
6410 }
6411
6412 sub qemu_img_convert {
6413     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6414
6415     my $storecfg = PVE::Storage::config();
6416     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6417     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6418
6419     if ($src_storeid && $dst_storeid) {
6420
6421         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6422
6423         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6424         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6425
6426         my $src_format = qemu_img_format($src_scfg, $src_volname);
6427         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6428
6429         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6430         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6431
6432         my $cmd = [];
6433         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6434         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6435         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6436         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6437         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6438         if ($is_zero_initialized) {
6439             push @$cmd, "zeroinit:$dst_path";
6440         } else {
6441             push @$cmd, $dst_path;
6442         }
6443
6444         my $parser = sub {
6445             my $line = shift;
6446             if($line =~ m/\((\S+)\/100\%\)/){
6447                 my $percent = $1;
6448                 my $transferred = int($size * $percent / 100);
6449                 my $remaining = $size - $transferred;
6450
6451                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6452             }
6453
6454         };
6455
6456         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6457         my $err = $@;
6458         die "copy failed: $err" if $err;
6459     }
6460 }
6461
6462 sub qemu_img_format {
6463     my ($scfg, $volname) = @_;
6464
6465     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6466         return $1;
6467     } else {
6468         return "raw";
6469     }
6470 }
6471
6472 sub qemu_drive_mirror {
6473     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6474
6475     $jobs = {} if !$jobs;
6476
6477     my $qemu_target;
6478     my $format;
6479     $jobs->{"drive-$drive"} = {};
6480
6481     if ($dst_volid =~ /^nbd:/) {
6482         $qemu_target = $dst_volid;
6483         $format = "nbd";
6484     } else {
6485         my $storecfg = PVE::Storage::config();
6486         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6487
6488         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6489
6490         $format = qemu_img_format($dst_scfg, $dst_volname);
6491
6492         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6493
6494         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6495     }
6496
6497     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6498     $opts->{format} = $format if $format;
6499
6500     print "drive mirror is starting for drive-$drive\n";
6501
6502     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6503
6504     if (my $err = $@) {
6505         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6506         die "mirroring error: $err";
6507     }
6508
6509     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6510 }
6511
6512 sub qemu_drive_mirror_monitor {
6513     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6514
6515     eval {
6516         my $err_complete = 0;
6517
6518         while (1) {
6519             die "storage migration timed out\n" if $err_complete > 300;
6520
6521             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6522
6523             my $running_mirror_jobs = {};
6524             foreach my $stat (@$stats) {
6525                 next if $stat->{type} ne 'mirror';
6526                 $running_mirror_jobs->{$stat->{device}} = $stat;
6527             }
6528
6529             my $readycounter = 0;
6530
6531             foreach my $job (keys %$jobs) {
6532
6533                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6534                     print "$job : finished\n";
6535                     delete $jobs->{$job};
6536                     next;
6537                 }
6538
6539                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6540
6541                 my $busy = $running_mirror_jobs->{$job}->{busy};
6542                 my $ready = $running_mirror_jobs->{$job}->{ready};
6543                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6544                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6545                     my $remaining = $total - $transferred;
6546                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6547
6548                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6549                 }
6550
6551                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6552             }
6553
6554             last if scalar(keys %$jobs) == 0;
6555
6556             if ($readycounter == scalar(keys %$jobs)) {
6557                 print "all mirroring jobs are ready \n";
6558                 last if $skipcomplete; #do the complete later
6559
6560                 if ($vmiddst && $vmiddst != $vmid) {
6561                     my $agent_running = $qga && qga_check_running($vmid);
6562                     if ($agent_running) {
6563                         print "freeze filesystem\n";
6564                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6565                     } else {
6566                         print "suspend vm\n";
6567                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6568                     }
6569
6570                     # if we clone a disk for a new target vm, we don't switch the disk
6571                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6572
6573                     if ($agent_running) {
6574                         print "unfreeze filesystem\n";
6575                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6576                     } else {
6577                         print "resume vm\n";
6578                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6579                     }
6580
6581                     last;
6582                 } else {
6583
6584                     foreach my $job (keys %$jobs) {
6585                         # try to switch the disk if source and destination are on the same guest
6586                         print "$job: Completing block job...\n";
6587
6588                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6589                         if ($@ =~ m/cannot be completed/) {
6590                             print "$job: Block job cannot be completed, try again.\n";
6591                             $err_complete++;
6592                         }else {
6593                             print "$job: Completed successfully.\n";
6594                             $jobs->{$job}->{complete} = 1;
6595                         }
6596                     }
6597                 }
6598             }
6599             sleep 1;
6600         }
6601     };
6602     my $err = $@;
6603
6604     if ($err) {
6605         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6606         die "mirroring error: $err";
6607     }
6608
6609 }
6610
6611 sub qemu_blockjobs_cancel {
6612     my ($vmid, $jobs) = @_;
6613
6614     foreach my $job (keys %$jobs) {
6615         print "$job: Cancelling block job\n";
6616         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6617         $jobs->{$job}->{cancel} = 1;
6618     }
6619
6620     while (1) {
6621         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6622
6623         my $running_jobs = {};
6624         foreach my $stat (@$stats) {
6625             $running_jobs->{$stat->{device}} = $stat;
6626         }
6627
6628         foreach my $job (keys %$jobs) {
6629
6630             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6631                 print "$job: Done.\n";
6632                 delete $jobs->{$job};
6633             }
6634         }
6635
6636         last if scalar(keys %$jobs) == 0;
6637
6638         sleep 1;
6639     }
6640 }
6641
6642 sub clone_disk {
6643     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6644         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6645
6646     my $newvolid;
6647
6648     if (!$full) {
6649         print "create linked clone of drive $drivename ($drive->{file})\n";
6650         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6651         push @$newvollist, $newvolid;
6652     } else {
6653
6654         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6655         $storeid = $storage if $storage;
6656
6657         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6658         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6659
6660         print "create full clone of drive $drivename ($drive->{file})\n";
6661         my $name = undef;
6662         if (drive_is_cloudinit($drive)) {
6663             $name = "vm-$newvmid-cloudinit";
6664             # cloudinit only supports raw and qcow2 atm:
6665             if ($dst_format eq 'qcow2') {
6666                 $name .= '.qcow2';
6667             } elsif ($dst_format ne 'raw') {
6668                 die "clone: unhandled format for cloudinit image\n";
6669             }
6670         }
6671         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6672         push @$newvollist, $newvolid;
6673
6674         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6675
6676         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6677         if (!$running || $snapname) {
6678             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6679         } else {
6680
6681             my $kvmver = get_running_qemu_version ($vmid);
6682             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6683                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6684                     if $drive->{iothread};
6685             }
6686
6687             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6688         }
6689     }
6690
6691     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6692
6693     my $disk = $drive;
6694     $disk->{format} = undef;
6695     $disk->{file} = $newvolid;
6696     $disk->{size} = $size;
6697
6698     return $disk;
6699 }
6700
6701 # this only works if VM is running
6702 sub get_current_qemu_machine {
6703     my ($vmid) = @_;
6704
6705     my $cmd = { execute => 'query-machines', arguments => {} };
6706     my $res = vm_qmp_command($vmid, $cmd);
6707
6708     my ($current, $default);
6709     foreach my $e (@$res) {
6710         $default = $e->{name} if $e->{'is-default'};
6711         $current = $e->{name} if $e->{'is-current'};
6712     }
6713
6714     # fallback to the default machine if current is not supported by qemu
6715     return $current || $default || 'pc';
6716 }
6717
6718 sub get_running_qemu_version {
6719     my ($vmid) = @_;
6720     my $cmd = { execute => 'query-version', arguments => {} };
6721     my $res = vm_qmp_command($vmid, $cmd);
6722     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6723 }
6724
6725 sub qemu_machine_feature_enabled {
6726     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6727
6728     my $current_major;
6729     my $current_minor;
6730
6731     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6732
6733         $current_major = $3;
6734         $current_minor = $4;
6735
6736     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6737
6738         $current_major = $1;
6739         $current_minor = $2;
6740     }
6741
6742     return 1 if $current_major > $version_major ||
6743                 ($current_major == $version_major &&
6744                  $current_minor >= $version_minor);
6745 }
6746
6747 sub qemu_machine_pxe {
6748     my ($vmid, $conf, $machine) = @_;
6749
6750     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6751
6752     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6753         $machine .= '.pxe';
6754     }
6755
6756     return $machine;
6757 }
6758
6759 sub qemu_use_old_bios_files {
6760     my ($machine_type) = @_;
6761
6762     return if !$machine_type;
6763
6764     my $use_old_bios_files = undef;
6765
6766     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6767         $machine_type = $1;
6768         $use_old_bios_files = 1;
6769     } else {
6770         my $kvmver = kvm_user_version();
6771         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6772         # load new efi bios files on migration. So this hack is required to allow
6773         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6774         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6775         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6776     }
6777
6778     return ($use_old_bios_files, $machine_type);
6779 }
6780
6781 sub create_efidisk($$$$$) {
6782     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6783
6784     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6785     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6786
6787     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6788     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6789     PVE::Storage::activate_volumes($storecfg, [$volid]);
6790
6791     my $path = PVE::Storage::path($storecfg, $volid);
6792     eval {
6793         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6794     };
6795     die "Copying EFI vars image failed: $@" if $@;
6796
6797     return ($volid, $vars_size);
6798 }
6799
6800 sub vm_iothreads_list {
6801     my ($vmid) = @_;
6802
6803     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6804
6805     my $iothreads = {};
6806     foreach my $iothread (@$res) {
6807         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6808     }
6809
6810     return $iothreads;
6811 }
6812
6813 sub scsihw_infos {
6814     my ($conf, $drive) = @_;
6815
6816     my $maxdev = 0;
6817
6818     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6819         $maxdev = 7;
6820     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6821         $maxdev = 1;
6822     } else {
6823         $maxdev = 256;
6824     }
6825
6826     my $controller = int($drive->{index} / $maxdev);
6827     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6828
6829     return ($maxdev, $controller, $controller_prefix);
6830 }
6831
6832 sub add_hyperv_enlightenments {
6833     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6834
6835     return if $winversion < 6;
6836     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6837
6838     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6839
6840     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6841         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6842         push @$cpuFlags , 'hv_vapic';
6843         push @$cpuFlags , 'hv_time';
6844     } else {
6845         push @$cpuFlags , 'hv_spinlocks=0xffff';
6846     }
6847
6848     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6849         push @$cpuFlags , 'hv_reset';
6850         push @$cpuFlags , 'hv_vpindex';
6851         push @$cpuFlags , 'hv_runtime';
6852     }
6853
6854     if ($winversion >= 7) {
6855         push @$cpuFlags , 'hv_relaxed';
6856
6857         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6858             push @$cpuFlags , 'hv_synic';
6859             push @$cpuFlags , 'hv_stimer';
6860         }
6861     }
6862 }
6863
6864 sub windows_version {
6865     my ($ostype) = @_;
6866
6867     return 0 if !$ostype;
6868
6869     my $winversion = 0;
6870
6871     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6872         $winversion = 5;
6873     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6874         $winversion = 6;
6875     } elsif ($ostype =~ m/^win(\d+)$/) {
6876         $winversion = $1;
6877     }
6878
6879     return $winversion;
6880 }
6881
6882 sub resolve_dst_disk_format {
6883         my ($storecfg, $storeid, $src_volname, $format) = @_;
6884         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6885
6886         if (!$format) {
6887             # if no target format is specified, use the source disk format as hint
6888             if ($src_volname) {
6889                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6890                 $format = qemu_img_format($scfg, $src_volname);
6891             } else {
6892                 return $defFormat;
6893             }
6894         }
6895
6896         # test if requested format is supported - else use default
6897         my $supported = grep { $_ eq $format } @$validFormats;
6898         $format = $defFormat if !$supported;
6899         return $format;
6900 }
6901
6902 sub resolve_first_disk {
6903     my $conf = shift;
6904     my @disks = PVE::QemuServer::valid_drive_names();
6905     my $firstdisk;
6906     foreach my $ds (reverse @disks) {
6907         next if !$conf->{$ds};
6908         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6909         next if PVE::QemuServer::drive_is_cdrom($disk);
6910         $firstdisk = $ds;
6911     }
6912     return $firstdisk;
6913 }
6914
6915 sub generate_uuid {
6916     my ($uuid, $uuid_str);
6917     UUID::generate($uuid);
6918     UUID::unparse($uuid, $uuid_str);
6919     return $uuid_str;
6920 }
6921
6922 sub generate_smbios1_uuid {
6923     return "uuid=".generate_uuid();
6924 }
6925
6926 sub nbd_stop {
6927     my ($vmid) = @_;
6928
6929     vm_mon_cmd($vmid, 'nbd-server-stop');
6930 }
6931
6932 # bash completion helper
6933
6934 sub complete_backup_archives {
6935     my ($cmdname, $pname, $cvalue) = @_;
6936
6937     my $cfg = PVE::Storage::config();
6938
6939     my $storeid;
6940
6941     if ($cvalue =~ m/^([^:]+):/) {
6942         $storeid = $1;
6943     }
6944
6945     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6946
6947     my $res = [];
6948     foreach my $id (keys %$data) {
6949         foreach my $item (@{$data->{$id}}) {
6950             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6951             push @$res, $item->{volid} if defined($item->{volid});
6952         }
6953     }
6954
6955     return $res;
6956 }
6957
6958 my $complete_vmid_full = sub {
6959     my ($running) = @_;
6960
6961     my $idlist = vmstatus();
6962
6963     my $res = [];
6964
6965     foreach my $id (keys %$idlist) {
6966         my $d = $idlist->{$id};
6967         if (defined($running)) {
6968             next if $d->{template};
6969             next if $running && $d->{status} ne 'running';
6970             next if !$running && $d->{status} eq 'running';
6971         }
6972         push @$res, $id;
6973
6974     }
6975     return $res;
6976 };
6977
6978 sub complete_vmid {
6979     return &$complete_vmid_full();
6980 }
6981
6982 sub complete_vmid_stopped {
6983     return &$complete_vmid_full(0);
6984 }
6985
6986 sub complete_vmid_running {
6987     return &$complete_vmid_full(1);
6988 }
6989
6990 sub complete_storage {
6991
6992     my $cfg = PVE::Storage::config();
6993     my $ids = $cfg->{ids};
6994
6995     my $res = [];
6996     foreach my $sid (keys %$ids) {
6997         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6998         next if !$ids->{$sid}->{content}->{images};
6999         push @$res, $sid;
7000     }
7001
7002     return $res;
7003 }
7004
7005 1;