PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use Time::HiRes qw(gettimeofday);
  40 use File::Copy qw(copy);
  41 use URI::Escape;
  42
  43 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  44 my $OVMF = {
  45     x86_64 => [
  46         "$EDK2_FW_BASE/OVMF_CODE.fd",
  47         "$EDK2_FW_BASE/OVMF_VARS.fd"
  48     ],
  49     aarch64 => [
  50         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  51         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  52     ],
  53 };
  54
  55 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  56
  57 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  58
  59 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  60
  61 # Note about locking: we use flock on the config file protect
  62 # against concurent actions.
  63 # Aditionaly, we have a 'lock' setting in the config file. This
  64 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  65 # allowed when such lock is set. But you can ignore this kind of
  66 # lock with the --skiplock flag.
  67
  68 cfs_register_file('/qemu-server/',
  69                   \&parse_vm_config,
  70                   \&write_vm_config);
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  73     description => "Some command save/restore state from this location.",
  74     type => 'string',
  75     maxLength => 128,
  76     optional => 1,
  77 });
  78
  79 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  80     description => "The name of the snapshot.",
  81     type => 'string', format => 'pve-configid',
  82     maxLength => 40,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $cpu_vendor_list = {
 123     # Intel CPUs
 124     486 => 'GenuineIntel',
 125     pentium => 'GenuineIntel',
 126     pentium2  => 'GenuineIntel',
 127     pentium3  => 'GenuineIntel',
 128     coreduo => 'GenuineIntel',
 129     core2duo => 'GenuineIntel',
 130     Conroe  => 'GenuineIntel',
 131     Penryn  => 'GenuineIntel',
 132     Nehalem  => 'GenuineIntel',
 133     'Nehalem-IBRS'  => 'GenuineIntel',
 134     Westmere => 'GenuineIntel',
 135     'Westmere-IBRS' => 'GenuineIntel',
 136     SandyBridge => 'GenuineIntel',
 137     'SandyBridge-IBRS' => 'GenuineIntel',
 138     IvyBridge => 'GenuineIntel',
 139     'IvyBridge-IBRS' => 'GenuineIntel',
 140     Haswell => 'GenuineIntel',
 141     'Haswell-IBRS' => 'GenuineIntel',
 142     'Haswell-noTSX' => 'GenuineIntel',
 143     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 144     Broadwell => 'GenuineIntel',
 145     'Broadwell-IBRS' => 'GenuineIntel',
 146     'Broadwell-noTSX' => 'GenuineIntel',
 147     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 148     'Skylake-Client' => 'GenuineIntel',
 149     'Skylake-Client-IBRS' => 'GenuineIntel',
 150     'Skylake-Server' => 'GenuineIntel',
 151     'Skylake-Server-IBRS' => 'GenuineIntel',
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 174
 175 my $cpu_fmt = {
 176     cputype => {
 177         description => "Emulated CPU type.",
 178         type => 'string',
 179         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 180         default => 'kvm64',
 181         default_key => 1,
 182     },
 183     hidden => {
 184         description => "Do not identify as a KVM virtual machine.",
 185         type => 'boolean',
 186         optional => 1,
 187         default => 0
 188     },
 189     flags => {
 190         description => "List of additional CPU flags separated by ';'."
 191                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 192                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 193         format_description => '+FLAG[;-FLAG...]',
 194         type => 'string',
 195         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 196         optional => 1,
 197     },
 198 };
 199
 200 my $watchdog_fmt = {
 201     model => {
 202         default_key => 1,
 203         type => 'string',
 204         enum => [qw(i6300esb ib700)],
 205         description => "Watchdog type to emulate.",
 206         default => 'i6300esb',
 207         optional => 1,
 208     },
 209     action => {
 210         type => 'string',
 211         enum => [qw(reset shutdown poweroff pause debug none)],
 212         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 213         optional => 1,
 214     },
 215 };
 216 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 217
 218 my $agent_fmt = {
 219     enabled => {
 220         description => "Enable/disable Qemu GuestAgent.",
 221         type => 'boolean',
 222         default => 0,
 223         default_key => 1,
 224     },
 225     fstrim_cloned_disks => {
 226         description => "Run fstrim after cloning/moving a disk.",
 227         type => 'boolean',
 228         optional => 1,
 229         default => 0
 230     },
 231 };
 232
 233 my $vga_fmt = {
 234     type => {
 235         description => "Select the VGA type.",
 236         type => 'string',
 237         default => 'std',
 238         optional => 1,
 239         default_key => 1,
 240         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 241     },
 242     memory => {
 243         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 244         type => 'integer',
 245         optional => 1,
 246         minimum => 4,
 247         maximum => 512,
 248     },
 249 };
 250
 251 my $confdesc = {
 252     onboot => {
 253         optional => 1,
 254         type => 'boolean',
 255         description => "Specifies whether a VM will be started during system bootup.",
 256         default => 0,
 257     },
 258     autostart => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Automatic restart after crash (currently ignored).",
 262         default => 0,
 263     },
 264     hotplug => {
 265         optional => 1,
 266         type => 'string', format => 'pve-hotplug-features',
 267         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 268         default => 'network,disk,usb',
 269     },
 270     reboot => {
 271         optional => 1,
 272         type => 'boolean',
 273         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 274         default => 1,
 275     },
 276     lock => {
 277         optional => 1,
 278         type => 'string',
 279         description => "Lock/unlock the VM.",
 280         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 281     },
 282     cpulimit => {
 283         optional => 1,
 284         type => 'number',
 285         description => "Limit of CPU usage.",
 286         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 287         minimum => 0,
 288         maximum => 128,
 289         default => 0,
 290     },
 291     cpuunits => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "CPU weight for a VM.",
 295         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 296         minimum => 2,
 297         maximum => 262144,
 298         default => 1024,
 299     },
 300     memory => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 304         minimum => 16,
 305         default => 512,
 306     },
 307     balloon => {
 308         optional => 1,
 309         type => 'integer',
 310         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 311         minimum => 0,
 312     },
 313     shares => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 317         minimum => 0,
 318         maximum => 50000,
 319         default => 1000,
 320     },
 321     keyboard => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 325                        "It should not be necessary to set it.",
 326         enum => PVE::Tools::kvmkeymaplist(),
 327         default => undef,
 328     },
 329     name => {
 330         optional => 1,
 331         type => 'string', format => 'dns-name',
 332         description => "Set a name for the VM. Only used on the configuration web interface.",
 333     },
 334     scsihw => {
 335         optional => 1,
 336         type => 'string',
 337         description => "SCSI controller model",
 338         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 339         default => 'lsi',
 340     },
 341     description => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 345     },
 346     ostype => {
 347         optional => 1,
 348         type => 'string',
 349         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 350         description => "Specify guest operating system.",
 351         verbose_description => <<EODESC,
 352 Specify guest operating system. This is used to enable special
 353 optimization/features for specific operating systems:
 354
 355 [horizontal]
 356 other;; unspecified OS
 357 wxp;; Microsoft Windows XP
 358 w2k;; Microsoft Windows 2000
 359 w2k3;; Microsoft Windows 2003
 360 w2k8;; Microsoft Windows 2008
 361 wvista;; Microsoft Windows Vista
 362 win7;; Microsoft Windows 7
 363 win8;; Microsoft Windows 8/2012/2012r2
 364 win10;; Microsoft Windows 10/2016
 365 l24;; Linux 2.4 Kernel
 366 l26;; Linux 2.6/3.X Kernel
 367 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 368 EODESC
 369     },
 370     boot => {
 371         optional => 1,
 372         type => 'string',
 373         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 374         pattern => '[acdn]{1,4}',
 375         default => 'cdn',
 376     },
 377     bootdisk => {
 378         optional => 1,
 379         type => 'string', format => 'pve-qm-bootdisk',
 380         description => "Enable booting from specified disk.",
 381         pattern => '(ide|sata|scsi|virtio)\d+',
 382     },
 383     smp => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPUs. Please use option -sockets instead.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     sockets => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of CPU sockets.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     cores => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "The number of cores per socket.",
 401         minimum => 1,
 402         default => 1,
 403     },
 404     numa => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Enable/disable NUMA.",
 408         default => 0,
 409     },
 410     hugepages => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Enable/disable hugepages memory.",
 414         enum => [qw(any 2 1024)],
 415     },
 416     vcpus => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "Number of hotplugged vcpus.",
 420         minimum => 1,
 421         default => 0,
 422     },
 423     acpi => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable ACPI.",
 427         default => 1,
 428     },
 429     agent => {
 430         optional => 1,
 431         description => "Enable/disable Qemu GuestAgent and its properties.",
 432         type => 'string',
 433         format => $agent_fmt,
 434     },
 435     kvm => {
 436         optional => 1,
 437         type => 'boolean',
 438         description => "Enable/disable KVM hardware virtualization.",
 439         default => 1,
 440     },
 441     tdf => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable time drift fix.",
 445         default => 0,
 446     },
 447     localtime => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 451     },
 452     freeze => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 456     },
 457     vga => {
 458         optional => 1,
 459         type => 'string', format => $vga_fmt,
 460         description => "Configure the VGA hardware.",
 461         verbose_description => "Configure the VGA Hardware. If you want to use ".
 462             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 463             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 464             "is 'std' for all OS types besides some Windows versions (XP and " .
 465             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 466             "display server. For win* OS you can select how many independent " .
 467             "displays you want, Linux guests can add displays them self.\n".
 468             "You can also run without any graphic card, using a serial device as terminal.",
 469     },
 470     watchdog => {
 471         optional => 1,
 472         type => 'string', format => 'pve-qm-watchdog',
 473         description => "Create a virtual hardware watchdog device.",
 474         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 475             " (by a guest action), the watchdog must be periodically polled " .
 476             "by an agent inside the guest or else the watchdog will reset " .
 477             "the guest (or execute the respective action specified)",
 478     },
 479     startdate => {
 480         optional => 1,
 481         type => 'string',
 482         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 483         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 484         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 485         default => 'now',
 486     },
 487     startup =>  get_standard_option('pve-startup-order'),
 488     template => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable Template.",
 492         default => 0,
 493     },
 494     args => {
 495         optional => 1,
 496         type => 'string',
 497         description => "Arbitrary arguments passed to kvm.",
 498         verbose_description => <<EODESCR,
 499 Arbitrary arguments passed to kvm, for example:
 500
 501 args: -no-reboot -no-hpet
 502
 503 NOTE: this option is for experts only.
 504 EODESCR
 505     },
 506     tablet => {
 507         optional => 1,
 508         type => 'boolean',
 509         default => 1,
 510         description => "Enable/disable the USB tablet device.",
 511         verbose_description => "Enable/disable the USB tablet device. This device is " .
 512             "usually needed to allow absolute mouse positioning with VNC. " .
 513             "Else the mouse runs out of sync with normal VNC clients. " .
 514             "If you're running lots of console-only guests on one host, " .
 515             "you may consider disabling this to save some context switches. " .
 516             "This is turned off by default if you use spice (-vga=qxl).",
 517     },
 518     migrate_speed => {
 519         optional => 1,
 520         type => 'integer',
 521         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 522         minimum => 0,
 523         default => 0,
 524     },
 525     migrate_downtime => {
 526         optional => 1,
 527         type => 'number',
 528         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 529         minimum => 0,
 530         default => 0.1,
 531     },
 532     cdrom => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-ide',
 535         typetext => '<volume>',
 536         description => "This is an alias for option -ide2",
 537     },
 538     cpu => {
 539         optional => 1,
 540         description => "Emulated CPU type.",
 541         type => 'string',
 542         format => $cpu_fmt,
 543     },
 544     parent => get_standard_option('pve-snapshot-name', {
 545         optional => 1,
 546         description => "Parent snapshot name. This is used internally, and should not be modified.",
 547     }),
 548     snaptime => {
 549         optional => 1,
 550         description => "Timestamp for snapshots.",
 551         type => 'integer',
 552         minimum => 0,
 553     },
 554     vmstate => {
 555         optional => 1,
 556         type => 'string', format => 'pve-volume-id',
 557         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 558     },
 559     vmstatestorage => get_standard_option('pve-storage-id', {
 560         description => "Default storage for VM state volumes/files.",
 561         optional => 1,
 562     }),
 563     runningmachine => get_standard_option('pve-qemu-machine', {
 564         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 565     }),
 566     machine => get_standard_option('pve-qemu-machine'),
 567     arch => {
 568         description => "Virtual processor architecture. Defaults to the host.",
 569         optional => 1,
 570         type => 'string',
 571         enum => [qw(x86_64 aarch64)],
 572     },
 573     smbios1 => {
 574         description => "Specify SMBIOS type 1 fields.",
 575         type => 'string', format => 'pve-qm-smbios1',
 576         maxLength => 256,
 577         optional => 1,
 578     },
 579     protection => {
 580         optional => 1,
 581         type => 'boolean',
 582         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 583         default => 0,
 584     },
 585     bios => {
 586         optional => 1,
 587         type => 'string',
 588         enum => [ qw(seabios ovmf) ],
 589         description => "Select BIOS implementation.",
 590         default => 'seabios',
 591     },
 592     vmgenid => {
 593         type => 'string',
 594         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 595         format_description => 'UUID',
 596         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 597         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 598             " 128-bit integer value identifier to the guest OS. This allows to".
 599             " notify the guest operating system when the virtual machine is".
 600             " executed with a different configuration (e.g. snapshot execution".
 601             " or creation from a template). The guest operating system notices".
 602             " the change, and is then able to react as appropriate by marking".
 603             " its copies of distributed databases as dirty, re-initializing its".
 604             " random number generator, etc.\n".
 605             "Note that auto-creation only works when done throug API/CLI create".
 606             " or update methods, but not when manually editing the config file.",
 607         default => "1 (autogenerated)",
 608         optional => 1,
 609     },
 610 };
 611
 612 my $confdesc_cloudinit = {
 613     citype => {
 614         optional => 1,
 615         type => 'string',
 616         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 617         enum => ['configdrive2', 'nocloud'],
 618     },
 619     ciuser => {
 620         optional => 1,
 621         type => 'string',
 622         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 623     },
 624     cipassword => {
 625         optional => 1,
 626         type => 'string',
 627         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 628     },
 629     searchdomain => {
 630         optional => 1,
 631         type => 'string',
 632         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 633     },
 634     nameserver => {
 635         optional => 1,
 636         type => 'string', format => 'address-list',
 637         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 638     },
 639     sshkeys => {
 640         optional => 1,
 641         type => 'string',
 642         format => 'urlencoded',
 643         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 644     },
 645 };
 646
 647 # what about other qemu settings ?
 648 #cpu => 'string',
 649 #machine => 'string',
 650 #fda => 'file',
 651 #fdb => 'file',
 652 #mtdblock => 'file',
 653 #sd => 'file',
 654 #pflash => 'file',
 655 #snapshot => 'bool',
 656 #bootp => 'file',
 657 ##tftp => 'dir',
 658 ##smb => 'dir',
 659 #kernel => 'file',
 660 #append => 'string',
 661 #initrd => 'file',
 662 ##soundhw => 'string',
 663
 664 while (my ($k, $v) = each %$confdesc) {
 665     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 666 }
 667
 668 my $MAX_IDE_DISKS = 4;
 669 my $MAX_SCSI_DISKS = 14;
 670 my $MAX_VIRTIO_DISKS = 16;
 671 my $MAX_SATA_DISKS = 6;
 672 my $MAX_USB_DEVICES = 5;
 673 my $MAX_NETS = 32;
 674 my $MAX_UNUSED_DISKS = 256;
 675 my $MAX_HOSTPCI_DEVICES = 4;
 676 my $MAX_SERIAL_PORTS = 4;
 677 my $MAX_PARALLEL_PORTS = 3;
 678 my $MAX_NUMA = 8;
 679
 680 my $numa_fmt = {
 681     cpus => {
 682         type => "string",
 683         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 684         description => "CPUs accessing this NUMA node.",
 685         format_description => "id[-id];...",
 686     },
 687     memory => {
 688         type => "number",
 689         description => "Amount of memory this NUMA node provides.",
 690         optional => 1,
 691     },
 692     hostnodes => {
 693         type => "string",
 694         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 695         description => "Host NUMA nodes to use.",
 696         format_description => "id[-id];...",
 697         optional => 1,
 698     },
 699     policy => {
 700         type => 'string',
 701         enum => [qw(preferred bind interleave)],
 702         description => "NUMA allocation policy.",
 703         optional => 1,
 704     },
 705 };
 706 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 707 my $numadesc = {
 708     optional => 1,
 709     type => 'string', format => $numa_fmt,
 710     description => "NUMA topology.",
 711 };
 712 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 713
 714 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 715     $confdesc->{"numa$i"} = $numadesc;
 716 }
 717
 718 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 719                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 720                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 721 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 722
 723 my $net_fmt_bridge_descr = <<__EOD__;
 724 Bridge to attach the network device to. The Proxmox VE standard bridge
 725 is called 'vmbr0'.
 726
 727 If you do not specify a bridge, we create a kvm user (NATed) network
 728 device, which provides DHCP and DNS services. The following addresses
 729 are used:
 730
 731  10.0.2.2   Gateway
 732  10.0.2.3   DNS Server
 733  10.0.2.4   SMB Server
 734
 735 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 736 __EOD__
 737
 738 my $net_fmt = {
 739     macaddr => {
 740         type => 'string',
 741         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 742         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 743         format_description => "XX:XX:XX:XX:XX:XX",
 744         optional => 1,
 745     },
 746     model => {
 747         type => 'string',
 748         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 749         enum => $nic_model_list,
 750         default_key => 1,
 751     },
 752     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 753     bridge => {
 754         type => 'string',
 755         description => $net_fmt_bridge_descr,
 756         format_description => 'bridge',
 757         optional => 1,
 758     },
 759     queues => {
 760         type => 'integer',
 761         minimum => 0, maximum => 16,
 762         description => 'Number of packet queues to be used on the device.',
 763         optional => 1,
 764     },
 765     rate => {
 766         type => 'number',
 767         minimum => 0,
 768         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 769         optional => 1,
 770     },
 771     tag => {
 772         type => 'integer',
 773         minimum => 1, maximum => 4094,
 774         description => 'VLAN tag to apply to packets on this interface.',
 775         optional => 1,
 776     },
 777     trunks => {
 778         type => 'string',
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => 'VLAN trunks to pass through this interface.',
 781         format_description => 'vlanid[;vlanid...]',
 782         optional => 1,
 783     },
 784     firewall => {
 785         type => 'boolean',
 786         description => 'Whether this interface should be protected by the firewall.',
 787         optional => 1,
 788     },
 789     link_down => {
 790         type => 'boolean',
 791         description => 'Whether this interface should be disconnected (like pulling the plug).',
 792         optional => 1,
 793     },
 794 };
 795
 796 my $netdesc = {
 797     optional => 1,
 798     type => 'string', format => $net_fmt,
 799     description => "Specify network devices.",
 800 };
 801
 802 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 803
 804 my $ipconfig_fmt = {
 805     ip => {
 806         type => 'string',
 807         format => 'pve-ipv4-config',
 808         format_description => 'IPv4Format/CIDR',
 809         description => 'IPv4 address in CIDR format.',
 810         optional => 1,
 811         default => 'dhcp',
 812     },
 813     gw => {
 814         type => 'string',
 815         format => 'ipv4',
 816         format_description => 'GatewayIPv4',
 817         description => 'Default gateway for IPv4 traffic.',
 818         optional => 1,
 819         requires => 'ip',
 820     },
 821     ip6 => {
 822         type => 'string',
 823         format => 'pve-ipv6-config',
 824         format_description => 'IPv6Format/CIDR',
 825         description => 'IPv6 address in CIDR format.',
 826         optional => 1,
 827         default => 'dhcp',
 828     },
 829     gw6 => {
 830         type => 'string',
 831         format => 'ipv6',
 832         format_description => 'GatewayIPv6',
 833         description => 'Default gateway for IPv6 traffic.',
 834         optional => 1,
 835         requires => 'ip6',
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 839 my $ipconfigdesc = {
 840     optional => 1,
 841     type => 'string', format => 'pve-qm-ipconfig',
 842     description => <<'EODESCR',
 843 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 844
 845 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 846
 847 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 848 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 849
 850 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 851 EODESCR
 852 };
 853 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 854
 855 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 856     $confdesc->{"net$i"} = $netdesc;
 857     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 858 }
 859
 860 foreach my $key (keys %$confdesc_cloudinit) {
 861     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 862 }
 863
 864 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 865 sub verify_volume_id_or_qm_path {
 866     my ($volid, $noerr) = @_;
 867
 868     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 869         return $volid;
 870     }
 871
 872     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 873     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 874     if ($@) {
 875         return undef if $noerr;
 876         die $@;
 877     }
 878     return $volid;
 879 }
 880
 881 my $drivename_hash;
 882
 883 my %drivedesc_base = (
 884     volume => { alias => 'file' },
 885     file => {
 886         type => 'string',
 887         format => 'pve-volume-id-or-qm-path',
 888         default_key => 1,
 889         format_description => 'volume',
 890         description => "The drive's backing volume.",
 891     },
 892     media => {
 893         type => 'string',
 894         enum => [qw(cdrom disk)],
 895         description => "The drive's media type.",
 896         default => 'disk',
 897         optional => 1
 898     },
 899     cyls => {
 900         type => 'integer',
 901         description => "Force the drive's physical geometry to have a specific cylinder count.",
 902         optional => 1
 903     },
 904     heads => {
 905         type => 'integer',
 906         description => "Force the drive's physical geometry to have a specific head count.",
 907         optional => 1
 908     },
 909     secs => {
 910         type => 'integer',
 911         description => "Force the drive's physical geometry to have a specific sector count.",
 912         optional => 1
 913     },
 914     trans => {
 915         type => 'string',
 916         enum => [qw(none lba auto)],
 917         description => "Force disk geometry bios translation mode.",
 918         optional => 1,
 919     },
 920     snapshot => {
 921         type => 'boolean',
 922         description => "Controls qemu's snapshot mode feature."
 923             . " If activated, changes made to the disk are temporary and will"
 924             . " be discarded when the VM is shutdown.",
 925         optional => 1,
 926     },
 927     cache => {
 928         type => 'string',
 929         enum => [qw(none writethrough writeback unsafe directsync)],
 930         description => "The drive's cache mode",
 931         optional => 1,
 932     },
 933     format => get_standard_option('pve-qm-image-format'),
 934     size => {
 935         type => 'string',
 936         format => 'disk-size',
 937         format_description => 'DiskSize',
 938         description => "Disk size. This is purely informational and has no effect.",
 939         optional => 1,
 940     },
 941     backup => {
 942         type => 'boolean',
 943         description => "Whether the drive should be included when making backups.",
 944         optional => 1,
 945     },
 946     replicate => {
 947         type => 'boolean',
 948         description => 'Whether the drive should considered for replication jobs.',
 949         optional => 1,
 950         default => 1,
 951     },
 952     rerror => {
 953         type => 'string',
 954         enum => [qw(ignore report stop)],
 955         description => 'Read error action.',
 956         optional => 1,
 957     },
 958     werror => {
 959         type => 'string',
 960         enum => [qw(enospc ignore report stop)],
 961         description => 'Write error action.',
 962         optional => 1,
 963     },
 964     aio => {
 965         type => 'string',
 966         enum => [qw(native threads)],
 967         description => 'AIO type to use.',
 968         optional => 1,
 969     },
 970     discard => {
 971         type => 'string',
 972         enum => [qw(ignore on)],
 973         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 974         optional => 1,
 975     },
 976     detect_zeroes => {
 977         type => 'boolean',
 978         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 979         optional => 1,
 980     },
 981     serial => {
 982         type => 'string',
 983         format => 'urlencoded',
 984         format_description => 'serial',
 985         maxLength => 20*3, # *3 since it's %xx url enoded
 986         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 987         optional => 1,
 988     },
 989     shared => {
 990         type => 'boolean',
 991         description => 'Mark this locally-managed volume as available on all nodes',
 992         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 993         optional => 1,
 994         default => 0,
 995     }
 996 );
 997
 998 my %iothread_fmt = ( iothread => {
 999         type => 'boolean',
1000         description => "Whether to use iothreads for this drive",
1001         optional => 1,
1002 });
1003
1004 my %model_fmt = (
1005     model => {
1006         type => 'string',
1007         format => 'urlencoded',
1008         format_description => 'model',
1009         maxLength => 40*3, # *3 since it's %xx url enoded
1010         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1011         optional => 1,
1012     },
1013 );
1014
1015 my %queues_fmt = (
1016     queues => {
1017         type => 'integer',
1018         description => "Number of queues.",
1019         minimum => 2,
1020         optional => 1
1021     }
1022 );
1023
1024 my %scsiblock_fmt = (
1025     scsiblock => {
1026         type => 'boolean',
1027         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1028         optional => 1,
1029         default => 0,
1030     },
1031 );
1032
1033 my %ssd_fmt = (
1034     ssd => {
1035         type => 'boolean',
1036         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1037         optional => 1,
1038     },
1039 );
1040
1041 my $add_throttle_desc = sub {
1042     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1043     my $d = {
1044         type => $type,
1045         format_description => $unit,
1046         description => "Maximum $what in $longunit.",
1047         optional => 1,
1048     };
1049     $d->{minimum} = $minimum if defined($minimum);
1050     $drivedesc_base{$key} = $d;
1051 };
1052 # throughput: (leaky bucket)
1053 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1054 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1056 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1057 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1059 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1060 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1061 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1062
1063 # pools: (pool of IO before throttling starts taking effect)
1064 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1065 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1067 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1068 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1069 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1070
1071 # burst lengths
1072 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1073 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1075 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1078
1079 # legacy support
1080 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1081 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1082 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1083 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1084
1085 my $ide_fmt = {
1086     %drivedesc_base,
1087     %model_fmt,
1088     %ssd_fmt,
1089 };
1090 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1091
1092 my $idedesc = {
1093     optional => 1,
1094     type => 'string', format => $ide_fmt,
1095     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1096 };
1097 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1098
1099 my $scsi_fmt = {
1100     %drivedesc_base,
1101     %iothread_fmt,
1102     %queues_fmt,
1103     %scsiblock_fmt,
1104     %ssd_fmt,
1105 };
1106 my $scsidesc = {
1107     optional => 1,
1108     type => 'string', format => $scsi_fmt,
1109     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1112
1113 my $sata_fmt = {
1114     %drivedesc_base,
1115     %ssd_fmt,
1116 };
1117 my $satadesc = {
1118     optional => 1,
1119     type => 'string', format => $sata_fmt,
1120     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1123
1124 my $virtio_fmt = {
1125     %drivedesc_base,
1126     %iothread_fmt,
1127 };
1128 my $virtiodesc = {
1129     optional => 1,
1130     type => 'string', format => $virtio_fmt,
1131     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1132 };
1133 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1134
1135 my $alldrive_fmt = {
1136     %drivedesc_base,
1137     %iothread_fmt,
1138     %model_fmt,
1139     %queues_fmt,
1140     %scsiblock_fmt,
1141     %ssd_fmt,
1142 };
1143
1144 my $efidisk_fmt = {
1145     volume => { alias => 'file' },
1146     file => {
1147         type => 'string',
1148         format => 'pve-volume-id-or-qm-path',
1149         default_key => 1,
1150         format_description => 'volume',
1151         description => "The drive's backing volume.",
1152     },
1153     format => get_standard_option('pve-qm-image-format'),
1154     size => {
1155         type => 'string',
1156         format => 'disk-size',
1157         format_description => 'DiskSize',
1158         description => "Disk size. This is purely informational and has no effect.",
1159         optional => 1,
1160     },
1161 };
1162
1163 my $efidisk_desc = {
1164     optional => 1,
1165     type => 'string', format => $efidisk_fmt,
1166     description => "Configure a Disk for storing EFI vars",
1167 };
1168
1169 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1170
1171 my $usb_fmt = {
1172     host => {
1173         default_key => 1,
1174         type => 'string', format => 'pve-qm-usb-device',
1175         format_description => 'HOSTUSBDEVICE|spice',
1176         description => <<EODESCR,
1177 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1178
1179  'bus-port(.port)*' (decimal numbers) or
1180  'vendor_id:product_id' (hexadeciaml numbers) or
1181  'spice'
1182
1183 You can use the 'lsusb -t' command to list existing usb devices.
1184
1185 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1186
1187 The value 'spice' can be used to add a usb redirection devices for spice.
1188 EODESCR
1189     },
1190     usb3 => {
1191         optional => 1,
1192         type => 'boolean',
1193         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1194         default => 0,
1195     },
1196 };
1197
1198 my $usbdesc = {
1199     optional => 1,
1200     type => 'string', format => $usb_fmt,
1201     description => "Configure an USB device (n is 0 to 4).",
1202 };
1203 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1204
1205 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1206 my $hostpci_fmt = {
1207     host => {
1208         default_key => 1,
1209         type => 'string',
1210         pattern => qr/$PCIRE(;$PCIRE)*/,
1211         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1212         description => <<EODESCR,
1213 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1214 of PCI virtual functions of the host. HOSTPCIID syntax is:
1215
1216 'bus:dev.func' (hexadecimal numbers)
1217
1218 You can us the 'lspci' command to list existing PCI devices.
1219 EODESCR
1220     },
1221     rombar => {
1222         type => 'boolean',
1223         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1224         optional => 1,
1225         default => 1,
1226     },
1227     romfile => {
1228         type => 'string',
1229         pattern => '[^,;]+',
1230         format_description => 'string',
1231         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1232         optional => 1,
1233     },
1234     pcie => {
1235         type => 'boolean',
1236         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1237         optional => 1,
1238         default => 0,
1239     },
1240     'x-vga' => {
1241         type => 'boolean',
1242         description =>  "Enable vfio-vga device support.",
1243         optional => 1,
1244         default => 0,
1245     },
1246     'mdev' => {
1247         type => 'string',
1248         format_description => 'string',
1249         pattern => '[^/\.:]+',
1250         optional => 1,
1251         description => <<EODESCR
1252 The type of mediated device to use.
1253 An instance of this type will be created on startup of the VM and
1254 will be cleaned up when the VM stops.
1255 EODESCR
1256     }
1257 };
1258 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1259
1260 my $hostpcidesc = {
1261         optional => 1,
1262         type => 'string', format => 'pve-qm-hostpci',
1263         description => "Map host PCI devices into guest.",
1264         verbose_description =>  <<EODESCR,
1265 Map host PCI devices into guest.
1266
1267 NOTE: This option allows direct access to host hardware. So it is no longer
1268 possible to migrate such machines - use with special care.
1269
1270 CAUTION: Experimental! User reported problems with this option.
1271 EODESCR
1272 };
1273 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1274
1275 my $serialdesc = {
1276         optional => 1,
1277         type => 'string',
1278         pattern => '(/dev/.+|socket)',
1279         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1280         verbose_description =>  <<EODESCR,
1281 Create a serial device inside the VM (n is 0 to 3), and pass through a
1282 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1283 host side (use 'qm terminal' to open a terminal connection).
1284
1285 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1286
1287 CAUTION: Experimental! User reported problems with this option.
1288 EODESCR
1289 };
1290
1291 my $paralleldesc= {
1292         optional => 1,
1293         type => 'string',
1294         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1295         description =>  "Map host parallel devices (n is 0 to 2).",
1296         verbose_description =>  <<EODESCR,
1297 Map host parallel devices (n is 0 to 2).
1298
1299 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1300
1301 CAUTION: Experimental! User reported problems with this option.
1302 EODESCR
1303 };
1304
1305 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1306     $confdesc->{"parallel$i"} = $paralleldesc;
1307 }
1308
1309 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1310     $confdesc->{"serial$i"} = $serialdesc;
1311 }
1312
1313 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1314     $confdesc->{"hostpci$i"} = $hostpcidesc;
1315 }
1316
1317 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1318     $drivename_hash->{"ide$i"} = 1;
1319     $confdesc->{"ide$i"} = $idedesc;
1320 }
1321
1322 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1323     $drivename_hash->{"sata$i"} = 1;
1324     $confdesc->{"sata$i"} = $satadesc;
1325 }
1326
1327 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1328     $drivename_hash->{"scsi$i"} = 1;
1329     $confdesc->{"scsi$i"} = $scsidesc ;
1330 }
1331
1332 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1333     $drivename_hash->{"virtio$i"} = 1;
1334     $confdesc->{"virtio$i"} = $virtiodesc;
1335 }
1336
1337 $drivename_hash->{efidisk0} = 1;
1338 $confdesc->{efidisk0} = $efidisk_desc;
1339
1340 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1341     $confdesc->{"usb$i"} = $usbdesc;
1342 }
1343
1344 my $unuseddesc = {
1345     optional => 1,
1346     type => 'string', format => 'pve-volume-id',
1347     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1348 };
1349
1350 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1351     $confdesc->{"unused$i"} = $unuseddesc;
1352 }
1353
1354 my $kvm_api_version = 0;
1355
1356 sub kvm_version {
1357     return $kvm_api_version if $kvm_api_version;
1358
1359     open my $fh, '<', '/dev/kvm'
1360         or return undef;
1361
1362     # 0xae00 => KVM_GET_API_VERSION
1363     $kvm_api_version = ioctl($fh, 0xae00, 0);
1364
1365     return $kvm_api_version;
1366 }
1367
1368 my $kvm_user_version;
1369
1370 sub kvm_user_version {
1371
1372     return $kvm_user_version if $kvm_user_version;
1373
1374     $kvm_user_version = 'unknown';
1375
1376     my $code = sub {
1377         my $line = shift;
1378         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1379             $kvm_user_version = $2;
1380         }
1381     };
1382
1383     eval { run_command("kvm -version", outfunc => $code); };
1384     warn $@ if $@;
1385
1386     return $kvm_user_version;
1387
1388 }
1389
1390 sub kernel_has_vhost_net {
1391     return -c '/dev/vhost-net';
1392 }
1393
1394 sub valid_drive_names {
1395     # order is important - used to autoselect boot disk
1396     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1397             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1398             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1399             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1400             'efidisk0');
1401 }
1402
1403 sub is_valid_drivename {
1404     my $dev = shift;
1405
1406     return defined($drivename_hash->{$dev});
1407 }
1408
1409 sub option_exists {
1410     my $key = shift;
1411     return defined($confdesc->{$key});
1412 }
1413
1414 sub nic_models {
1415     return $nic_model_list;
1416 }
1417
1418 sub os_list_description {
1419
1420     return {
1421         other => 'Other',
1422         wxp => 'Windows XP',
1423         w2k => 'Windows 2000',
1424         w2k3 =>, 'Windows 2003',
1425         w2k8 => 'Windows 2008',
1426         wvista => 'Windows Vista',
1427         win7 => 'Windows 7',
1428         win8 => 'Windows 8/2012',
1429         win10 => 'Windows 10/2016',
1430         l24 => 'Linux 2.4',
1431         l26 => 'Linux 2.6',
1432     };
1433 }
1434
1435 my $cdrom_path;
1436
1437 sub get_cdrom_path {
1438
1439     return  $cdrom_path if $cdrom_path;
1440
1441     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1442     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1443     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1444 }
1445
1446 sub get_iso_path {
1447     my ($storecfg, $vmid, $cdrom) = @_;
1448
1449     if ($cdrom eq 'cdrom') {
1450         return get_cdrom_path();
1451     } elsif ($cdrom eq 'none') {
1452         return '';
1453     } elsif ($cdrom =~ m|^/|) {
1454         return $cdrom;
1455     } else {
1456         return PVE::Storage::path($storecfg, $cdrom);
1457     }
1458 }
1459
1460 # try to convert old style file names to volume IDs
1461 sub filename_to_volume_id {
1462     my ($vmid, $file, $media) = @_;
1463
1464      if (!($file eq 'none' || $file eq 'cdrom' ||
1465           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1466
1467         return undef if $file =~ m|/|;
1468
1469         if ($media && $media eq 'cdrom') {
1470             $file = "local:iso/$file";
1471         } else {
1472             $file = "local:$vmid/$file";
1473         }
1474     }
1475
1476     return $file;
1477 }
1478
1479 sub verify_media_type {
1480     my ($opt, $vtype, $media) = @_;
1481
1482     return if !$media;
1483
1484     my $etype;
1485     if ($media eq 'disk') {
1486         $etype = 'images';
1487     } elsif ($media eq 'cdrom') {
1488         $etype = 'iso';
1489     } else {
1490         die "internal error";
1491     }
1492
1493     return if ($vtype eq $etype);
1494
1495     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1496 }
1497
1498 sub cleanup_drive_path {
1499     my ($opt, $storecfg, $drive) = @_;
1500
1501     # try to convert filesystem paths to volume IDs
1502
1503     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1504         ($drive->{file} !~ m|^/dev/.+|) &&
1505         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1506         ($drive->{file} !~ m/^\d+$/)) {
1507         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1508         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1509         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1510         verify_media_type($opt, $vtype, $drive->{media});
1511         $drive->{file} = $volid;
1512     }
1513
1514     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1515 }
1516
1517 sub parse_hotplug_features {
1518     my ($data) = @_;
1519
1520     my $res = {};
1521
1522     return $res if $data eq '0';
1523
1524     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1525
1526     foreach my $feature (PVE::Tools::split_list($data)) {
1527         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1528             $res->{$1} = 1;
1529         } else {
1530             die "invalid hotplug feature '$feature'\n";
1531         }
1532     }
1533     return $res;
1534 }
1535
1536 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1537 sub pve_verify_hotplug_features {
1538     my ($value, $noerr) = @_;
1539
1540     return $value if parse_hotplug_features($value);
1541
1542     return undef if $noerr;
1543
1544     die "unable to parse hotplug option\n";
1545 }
1546
1547 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1548 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1549 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1550 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1551 #        [,iothread=on][,serial=serial][,model=model]
1552
1553 sub parse_drive {
1554     my ($key, $data) = @_;
1555
1556     my ($interface, $index);
1557
1558     if ($key =~ m/^([^\d]+)(\d+)$/) {
1559         $interface = $1;
1560         $index = $2;
1561     } else {
1562         return undef;
1563     }
1564
1565     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1566                                      : $confdesc->{$key}->{format};
1567     if (!$desc) {
1568         warn "invalid drive key: $key\n";
1569         return undef;
1570     }
1571     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1572     return undef if !$res;
1573     $res->{interface} = $interface;
1574     $res->{index} = $index;
1575
1576     my $error = 0;
1577     foreach my $opt (qw(bps bps_rd bps_wr)) {
1578         if (my $bps = defined(delete $res->{$opt})) {
1579             if (defined($res->{"m$opt"})) {
1580                 warn "both $opt and m$opt specified\n";
1581                 ++$error;
1582                 next;
1583             }
1584             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1585         }
1586     }
1587
1588     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1589     for my $requirement (
1590         [mbps_max => 'mbps'],
1591         [mbps_rd_max => 'mbps_rd'],
1592         [mbps_wr_max => 'mbps_wr'],
1593         [miops_max => 'miops'],
1594         [miops_rd_max => 'miops_rd'],
1595         [miops_wr_max => 'miops_wr'],
1596         [bps_max_length => 'mbps_max'],
1597         [bps_rd_max_length => 'mbps_rd_max'],
1598         [bps_wr_max_length => 'mbps_wr_max'],
1599         [iops_max_length => 'iops_max'],
1600         [iops_rd_max_length => 'iops_rd_max'],
1601         [iops_wr_max_length => 'iops_wr_max']) {
1602         my ($option, $requires) = @$requirement;
1603         if ($res->{$option} && !$res->{$requires}) {
1604             warn "$option requires $requires\n";
1605             ++$error;
1606         }
1607     }
1608
1609     return undef if $error;
1610
1611     return undef if $res->{mbps_rd} && $res->{mbps};
1612     return undef if $res->{mbps_wr} && $res->{mbps};
1613     return undef if $res->{iops_rd} && $res->{iops};
1614     return undef if $res->{iops_wr} && $res->{iops};
1615
1616     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1617         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1618         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1619         return undef if $res->{interface} eq 'virtio';
1620     }
1621
1622     if (my $size = $res->{size}) {
1623         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1624     }
1625
1626     return $res;
1627 }
1628
1629 sub print_drive {
1630     my ($vmid, $drive) = @_;
1631     my $data = { %$drive };
1632     delete $data->{$_} for qw(index interface);
1633     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1634 }
1635
1636 sub scsi_inquiry {
1637     my($fh, $noerr) = @_;
1638
1639     my $SG_IO = 0x2285;
1640     my $SG_GET_VERSION_NUM = 0x2282;
1641
1642     my $versionbuf = "\x00" x 8;
1643     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1644     if (!$ret) {
1645         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1646         return undef;
1647     }
1648     my $version = unpack("I", $versionbuf);
1649     if ($version < 30000) {
1650         die "scsi generic interface too old\n"  if !$noerr;
1651         return undef;
1652     }
1653
1654     my $buf = "\x00" x 36;
1655     my $sensebuf = "\x00" x 8;
1656     my $cmd = pack("C x3 C x1", 0x12, 36);
1657
1658     # see /usr/include/scsi/sg.h
1659     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1660
1661     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1662                       length($sensebuf), 0, length($buf), $buf,
1663                       $cmd, $sensebuf, 6000);
1664
1665     $ret = ioctl($fh, $SG_IO, $packet);
1666     if (!$ret) {
1667         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1668         return undef;
1669     }
1670
1671     my @res = unpack($sg_io_hdr_t, $packet);
1672     if ($res[17] || $res[18]) {
1673         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1674         return undef;
1675     }
1676
1677     my $res = {};
1678     (my $byte0, my $byte1, $res->{vendor},
1679      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1680
1681     $res->{removable} = $byte1 & 128 ? 1 : 0;
1682     $res->{type} = $byte0 & 31;
1683
1684     return $res;
1685 }
1686
1687 sub path_is_scsi {
1688     my ($path) = @_;
1689
1690     my $fh = IO::File->new("+<$path") || return undef;
1691     my $res = scsi_inquiry($fh, 1);
1692     close($fh);
1693
1694     return $res;
1695 }
1696
1697 sub machine_type_is_q35 {
1698     my ($conf) = @_;
1699
1700     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1701 }
1702
1703 sub print_tabletdevice_full {
1704     my ($conf, $arch) = @_;
1705
1706     my $q35 = machine_type_is_q35($conf);
1707
1708     # we use uhci for old VMs because tablet driver was buggy in older qemu
1709     my $usbbus;
1710     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1711         $usbbus = 'ehci';
1712     } else {
1713         $usbbus = 'uhci';
1714     }
1715
1716     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1717 }
1718
1719 sub print_keyboarddevice_full {
1720     my ($conf, $arch, $machine) = @_;
1721
1722     return undef if $arch ne 'aarch64';
1723
1724     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1725 }
1726
1727 sub print_drivedevice_full {
1728     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1729
1730     my $device = '';
1731     my $maxdev = 0;
1732
1733     if ($drive->{interface} eq 'virtio') {
1734         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1735         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1736         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1737     } elsif ($drive->{interface} eq 'scsi') {
1738
1739         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1740         my $unit = $drive->{index} % $maxdev;
1741         my $devicetype = 'hd';
1742         my $path = '';
1743         if (drive_is_cdrom($drive)) {
1744             $devicetype = 'cd';
1745         } else {
1746             if ($drive->{file} =~ m|^/|) {
1747                 $path = $drive->{file};
1748                 if (my $info = path_is_scsi($path)) {
1749                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1750                         $devicetype = 'block';
1751                     } elsif ($info->{type} == 1) { # tape
1752                         $devicetype = 'generic';
1753                     }
1754                 }
1755             } else {
1756                  $path = PVE::Storage::path($storecfg, $drive->{file});
1757             }
1758
1759             if($path =~ m/^iscsi\:\/\//){
1760                 $devicetype = 'generic';
1761             }
1762         }
1763
1764         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1765             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1766         } else {
1767             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1768         }
1769
1770         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1771             $device .= ",rotation_rate=1";
1772         }
1773
1774     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1775         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1776         my $controller = int($drive->{index} / $maxdev);
1777         my $unit = $drive->{index} % $maxdev;
1778         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1779
1780         $device = "ide-$devicetype";
1781         if ($drive->{interface} eq 'ide') {
1782             $device .= ",bus=ide.$controller,unit=$unit";
1783         } else {
1784             $device .= ",bus=ahci$controller.$unit";
1785         }
1786         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1787
1788         if ($devicetype eq 'hd') {
1789             if (my $model = $drive->{model}) {
1790                 $model = URI::Escape::uri_unescape($model);
1791                 $device .= ",model=$model";
1792             }
1793             if ($drive->{ssd}) {
1794                 $device .= ",rotation_rate=1";
1795             }
1796         }
1797     } elsif ($drive->{interface} eq 'usb') {
1798         die "implement me";
1799         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1800     } else {
1801         die "unsupported interface type";
1802     }
1803
1804     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1805
1806     if (my $serial = $drive->{serial}) {
1807         $serial = URI::Escape::uri_unescape($serial);
1808         $device .= ",serial=$serial";
1809     }
1810
1811
1812     return $device;
1813 }
1814
1815 sub get_initiator_name {
1816     my $initiator;
1817
1818     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1819     while (defined(my $line = <$fh>)) {
1820         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1821         $initiator = $1;
1822         last;
1823     }
1824     $fh->close();
1825
1826     return $initiator;
1827 }
1828
1829 sub print_drive_full {
1830     my ($storecfg, $vmid, $drive) = @_;
1831
1832     my $path;
1833     my $volid = $drive->{file};
1834     my $format;
1835
1836     if (drive_is_cdrom($drive)) {
1837         $path = get_iso_path($storecfg, $vmid, $volid);
1838     } else {
1839         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1840         if ($storeid) {
1841             $path = PVE::Storage::path($storecfg, $volid);
1842             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1843             $format = qemu_img_format($scfg, $volname);
1844         } else {
1845             $path = $volid;
1846             $format = "raw";
1847         }
1848    }
1849
1850     my $opts = '';
1851     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1852     foreach my $o (@qemu_drive_options) {
1853         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1854     }
1855
1856     # snapshot only accepts on|off
1857     if (defined($drive->{snapshot})) {
1858         my $v = $drive->{snapshot} ? 'on' : 'off';
1859         $opts .= ",snapshot=$v";
1860     }
1861
1862     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1863         my ($dir, $qmpname) = @$type;
1864         if (my $v = $drive->{"mbps$dir"}) {
1865             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1866         }
1867         if (my $v = $drive->{"mbps${dir}_max"}) {
1868             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1869         }
1870         if (my $v = $drive->{"bps${dir}_max_length"}) {
1871             $opts .= ",throttling.bps$qmpname-max-length=$v";
1872         }
1873         if (my $v = $drive->{"iops${dir}"}) {
1874             $opts .= ",throttling.iops$qmpname=$v";
1875         }
1876         if (my $v = $drive->{"iops${dir}_max"}) {
1877             $opts .= ",throttling.iops$qmpname-max=$v";
1878         }
1879         if (my $v = $drive->{"iops${dir}_max_length"}) {
1880             $opts .= ",throttling.iops$qmpname-max-length=$v";
1881         }
1882     }
1883
1884     $opts .= ",format=$format" if $format && !$drive->{format};
1885
1886     my $cache_direct = 0;
1887
1888     if (my $cache = $drive->{cache}) {
1889         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1890     } elsif (!drive_is_cdrom($drive)) {
1891         $opts .= ",cache=none";
1892         $cache_direct = 1;
1893     }
1894
1895     # aio native works only with O_DIRECT
1896     if (!$drive->{aio}) {
1897         if($cache_direct) {
1898             $opts .= ",aio=native";
1899         } else {
1900             $opts .= ",aio=threads";
1901         }
1902     }
1903
1904     if (!drive_is_cdrom($drive)) {
1905         my $detectzeroes;
1906         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1907             $detectzeroes = 'off';
1908         } elsif ($drive->{discard}) {
1909             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1910         } else {
1911             # This used to be our default with discard not being specified:
1912             $detectzeroes = 'on';
1913         }
1914         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1915     }
1916
1917     my $pathinfo = $path ? "file=$path," : '';
1918
1919     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1920 }
1921
1922 sub print_netdevice_full {
1923     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1924
1925     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1926
1927     my $device = $net->{model};
1928     if ($net->{model} eq 'virtio') {
1929          $device = 'virtio-net-pci';
1930      };
1931
1932     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1933     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1934     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1935         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1936         my $vectors = $net->{queues} * 2 + 2;
1937         $tmpstr .= ",vectors=$vectors,mq=on";
1938     }
1939     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1940
1941     if ($use_old_bios_files) {
1942         my $romfile;
1943         if ($device eq 'virtio-net-pci') {
1944             $romfile = 'pxe-virtio.rom';
1945         } elsif ($device eq 'e1000') {
1946             $romfile = 'pxe-e1000.rom';
1947         } elsif ($device eq 'ne2k') {
1948             $romfile = 'pxe-ne2k_pci.rom';
1949         } elsif ($device eq 'pcnet') {
1950             $romfile = 'pxe-pcnet.rom';
1951         } elsif ($device eq 'rtl8139') {
1952             $romfile = 'pxe-rtl8139.rom';
1953         }
1954         $tmpstr .= ",romfile=$romfile" if $romfile;
1955     }
1956
1957     return $tmpstr;
1958 }
1959
1960 sub print_netdev_full {
1961     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1962
1963     my $i = '';
1964     if ($netid =~ m/^net(\d+)$/) {
1965         $i = int($1);
1966     }
1967
1968     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1969
1970     my $ifname = "tap${vmid}i$i";
1971
1972     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1973     die "interface name '$ifname' is too long (max 15 character)\n"
1974         if length($ifname) >= 16;
1975
1976     my $vhostparam = '';
1977     if (is_native($arch)) {
1978         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1979     }
1980
1981     my $vmname = $conf->{name} || "vm$vmid";
1982
1983     my $netdev = "";
1984     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1985
1986     if ($net->{bridge}) {
1987         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1988     } else {
1989         $netdev = "type=user,id=$netid,hostname=$vmname";
1990     }
1991
1992     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1993
1994     return $netdev;
1995 }
1996
1997
1998 sub print_cpu_device {
1999     my ($conf, $id) = @_;
2000
2001     my $kvm = $conf->{kvm} // 1;
2002     my $cpu = $kvm ? "kvm64" : "qemu64";
2003     if (my $cputype = $conf->{cpu}) {
2004         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2005             or die "Cannot parse cpu description: $cputype\n";
2006         $cpu = $cpuconf->{cputype};
2007     }
2008
2009     my $cores = $conf->{cores} || 1;
2010
2011     my $current_core = ($id - 1) % $cores;
2012     my $current_socket = int(($id - 1 - $current_core)/$cores);
2013
2014     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2015 }
2016
2017 my $vga_map = {
2018     'cirrus' => 'cirrus-vga',
2019     'std' => 'VGA',
2020     'vmware' => 'vmware-svga',
2021     'virtio' => 'virtio-vga',
2022 };
2023
2024 sub print_vga_device {
2025     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2026
2027     my $type = $vga_map->{$vga->{type}};
2028     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2029         $type = 'virtio-gpu';
2030     }
2031     my $vgamem_mb = $vga->{memory};
2032     if ($qxlnum) {
2033         $type = $id ? 'qxl' : 'qxl-vga';
2034     }
2035     die "no devicetype for $vga->{type}\n" if !$type;
2036
2037     my $memory = "";
2038     if ($vgamem_mb) {
2039         if ($vga->{type} eq 'virtio') {
2040             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2041             $memory = ",max_hostmem=$bytes";
2042         } elsif ($qxlnum) {
2043             # from https://www.spice-space.org/multiple-monitors.html
2044             $memory = ",vgamem_mb=$vga->{memory}";
2045             my $ram = $vgamem_mb * 4;
2046             my $vram = $vgamem_mb * 2;
2047             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2048         } else {
2049             $memory = ",vgamem_mb=$vga->{memory}";
2050         }
2051     } elsif ($qxlnum && $id) {
2052         $memory = ",ram_size=67108864,vram_size=33554432";
2053     }
2054
2055     my $q35 = machine_type_is_q35($conf);
2056     my $vgaid = "vga" . ($id // '');
2057     my $pciaddr;
2058
2059     if ($q35 && $vgaid eq 'vga') {
2060         # the first display uses pcie.0 bus on q35 machines
2061         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2062     } else {
2063         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2064     }
2065
2066     return "$type,id=${vgaid}${memory}${pciaddr}";
2067 }
2068
2069 sub drive_is_cloudinit {
2070     my ($drive) = @_;
2071     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2072 }
2073
2074 sub drive_is_cdrom {
2075     my ($drive, $exclude_cloudinit) = @_;
2076
2077     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2078
2079     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2080
2081 }
2082
2083 sub parse_number_sets {
2084     my ($set) = @_;
2085     my $res = [];
2086     foreach my $part (split(/;/, $set)) {
2087         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2088             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2089             push @$res, [ $1, $2 ];
2090         } else {
2091             die "invalid range: $part\n";
2092         }
2093     }
2094     return $res;
2095 }
2096
2097 sub parse_numa {
2098     my ($data) = @_;
2099
2100     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2101     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2102     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2103     return $res;
2104 }
2105
2106 sub parse_hostpci {
2107     my ($value) = @_;
2108
2109     return undef if !$value;
2110
2111     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2112
2113     my @idlist = split(/;/, $res->{host});
2114     delete $res->{host};
2115     foreach my $id (@idlist) {
2116         if ($id =~ m/\./) { # full id 00:00.1
2117             push @{$res->{pciid}}, {
2118                 id => $id,
2119             };
2120         } else { # partial id 00:00
2121             $res->{pciid} = PVE::SysFSTools::lspci($id);
2122         }
2123     }
2124     return $res;
2125 }
2126
2127 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2128 sub parse_net {
2129     my ($data) = @_;
2130
2131     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2132     if ($@) {
2133         warn $@;
2134         return undef;
2135     }
2136     if (!defined($res->{macaddr})) {
2137         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2138         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2139     }
2140     return $res;
2141 }
2142
2143 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2144 sub parse_ipconfig {
2145     my ($data) = @_;
2146
2147     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2148     if ($@) {
2149         warn $@;
2150         return undef;
2151     }
2152
2153     if ($res->{gw} && !$res->{ip}) {
2154         warn 'gateway specified without specifying an IP address';
2155         return undef;
2156     }
2157     if ($res->{gw6} && !$res->{ip6}) {
2158         warn 'IPv6 gateway specified without specifying an IPv6 address';
2159         return undef;
2160     }
2161     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2162         warn 'gateway specified together with DHCP';
2163         return undef;
2164     }
2165     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2166         # gw6 + auto/dhcp
2167         warn "IPv6 gateway specified together with $res->{ip6} address";
2168         return undef;
2169     }
2170
2171     if (!$res->{ip} && !$res->{ip6}) {
2172         return { ip => 'dhcp', ip6 => 'dhcp' };
2173     }
2174
2175     return $res;
2176 }
2177
2178 sub print_net {
2179     my $net = shift;
2180
2181     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2182 }
2183
2184 sub add_random_macs {
2185     my ($settings) = @_;
2186
2187     foreach my $opt (keys %$settings) {
2188         next if $opt !~ m/^net(\d+)$/;
2189         my $net = parse_net($settings->{$opt});
2190         next if !$net;
2191         $settings->{$opt} = print_net($net);
2192     }
2193 }
2194
2195 sub vm_is_volid_owner {
2196     my ($storecfg, $vmid, $volid) = @_;
2197
2198     if ($volid !~  m|^/|) {
2199         my ($path, $owner);
2200         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2201         if ($owner && ($owner == $vmid)) {
2202             return 1;
2203         }
2204     }
2205
2206     return undef;
2207 }
2208
2209 sub split_flagged_list {
2210     my $text = shift || '';
2211     $text =~ s/[,;]/ /g;
2212     $text =~ s/^\s+//;
2213     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2214 }
2215
2216 sub join_flagged_list {
2217     my ($how, $lst) = @_;
2218     join $how, map { $lst->{$_} . $_ } keys %$lst;
2219 }
2220
2221 sub vmconfig_delete_pending_option {
2222     my ($conf, $key, $force) = @_;
2223
2224     delete $conf->{pending}->{$key};
2225     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2226     $pending_delete_hash->{$key} = $force ? '!' : '';
2227     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2228 }
2229
2230 sub vmconfig_undelete_pending_option {
2231     my ($conf, $key) = @_;
2232
2233     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2234     delete $pending_delete_hash->{$key};
2235
2236     if (%$pending_delete_hash) {
2237         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2238     } else {
2239         delete $conf->{pending}->{delete};
2240     }
2241 }
2242
2243 sub vmconfig_register_unused_drive {
2244     my ($storecfg, $vmid, $conf, $drive) = @_;
2245
2246     if (drive_is_cloudinit($drive)) {
2247         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2248         warn $@ if $@;
2249     } elsif (!drive_is_cdrom($drive)) {
2250         my $volid = $drive->{file};
2251         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2252             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2253         }
2254     }
2255 }
2256
2257 sub vmconfig_cleanup_pending {
2258     my ($conf) = @_;
2259
2260     # remove pending changes when nothing changed
2261     my $changes;
2262     foreach my $opt (keys %{$conf->{pending}}) {
2263         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2264             $changes = 1;
2265             delete $conf->{pending}->{$opt};
2266         }
2267     }
2268
2269     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2270     my $pending_delete_hash = {};
2271     while (my ($opt, $force) = each %$current_delete_hash) {
2272         if (defined($conf->{$opt})) {
2273             $pending_delete_hash->{$opt} = $force;
2274         } else {
2275             $changes = 1;
2276         }
2277     }
2278
2279     if (%$pending_delete_hash) {
2280         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2281     } else {
2282         delete $conf->{pending}->{delete};
2283     }
2284
2285     return $changes;
2286 }
2287
2288 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2289 my $smbios1_fmt = {
2290     uuid => {
2291         type => 'string',
2292         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2293         format_description => 'UUID',
2294         description => "Set SMBIOS1 UUID.",
2295         optional => 1,
2296     },
2297     version => {
2298         type => 'string',
2299         pattern => '\S+',
2300         format_description => 'string',
2301         description => "Set SMBIOS1 version.",
2302         optional => 1,
2303     },
2304     serial => {
2305         type => 'string',
2306         pattern => '\S+',
2307         format_description => 'string',
2308         description => "Set SMBIOS1 serial number.",
2309         optional => 1,
2310     },
2311     manufacturer => {
2312         type => 'string',
2313         pattern => '\S+',
2314         format_description => 'string',
2315         description => "Set SMBIOS1 manufacturer.",
2316         optional => 1,
2317     },
2318     product => {
2319         type => 'string',
2320         pattern => '\S+',
2321         format_description => 'string',
2322         description => "Set SMBIOS1 product ID.",
2323         optional => 1,
2324     },
2325     sku => {
2326         type => 'string',
2327         pattern => '\S+',
2328         format_description => 'string',
2329         description => "Set SMBIOS1 SKU string.",
2330         optional => 1,
2331     },
2332     family => {
2333         type => 'string',
2334         pattern => '\S+',
2335         format_description => 'string',
2336         description => "Set SMBIOS1 family string.",
2337         optional => 1,
2338     },
2339 };
2340
2341 sub parse_smbios1 {
2342     my ($data) = @_;
2343
2344     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2345     warn $@ if $@;
2346     return $res;
2347 }
2348
2349 sub print_smbios1 {
2350     my ($smbios1) = @_;
2351     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2352 }
2353
2354 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2355
2356 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2357 sub verify_bootdisk {
2358     my ($value, $noerr) = @_;
2359
2360     return $value if is_valid_drivename($value);
2361
2362     return undef if $noerr;
2363
2364     die "invalid boot disk '$value'\n";
2365 }
2366
2367 sub parse_watchdog {
2368     my ($value) = @_;
2369
2370     return undef if !$value;
2371
2372     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2373     warn $@ if $@;
2374     return $res;
2375 }
2376
2377 sub parse_guest_agent {
2378     my ($value) = @_;
2379
2380     return {} if !defined($value->{agent});
2381
2382     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2383     warn $@ if $@;
2384
2385     # if the agent is disabled ignore the other potentially set properties
2386     return {} if !$res->{enabled};
2387     return $res;
2388 }
2389
2390 sub parse_vga {
2391     my ($value) = @_;
2392
2393     return {} if !$value;
2394     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2395     warn $@ if $@;
2396     return $res;
2397 }
2398
2399 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2400 sub verify_usb_device {
2401     my ($value, $noerr) = @_;
2402
2403     return $value if parse_usb_device($value);
2404
2405     return undef if $noerr;
2406
2407     die "unable to parse usb device\n";
2408 }
2409
2410 # add JSON properties for create and set function
2411 sub json_config_properties {
2412     my $prop = shift;
2413
2414     foreach my $opt (keys %$confdesc) {
2415         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2416         $prop->{$opt} = $confdesc->{$opt};
2417     }
2418
2419     return $prop;
2420 }
2421
2422 # return copy of $confdesc_cloudinit to generate documentation
2423 sub cloudinit_config_properties {
2424
2425     return dclone($confdesc_cloudinit);
2426 }
2427
2428 sub check_type {
2429     my ($key, $value) = @_;
2430
2431     die "unknown setting '$key'\n" if !$confdesc->{$key};
2432
2433     my $type = $confdesc->{$key}->{type};
2434
2435     if (!defined($value)) {
2436         die "got undefined value\n";
2437     }
2438
2439     if ($value =~ m/[\n\r]/) {
2440         die "property contains a line feed\n";
2441     }
2442
2443     if ($type eq 'boolean') {
2444         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2445         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2446         die "type check ('boolean') failed - got '$value'\n";
2447     } elsif ($type eq 'integer') {
2448         return int($1) if $value =~ m/^(\d+)$/;
2449         die "type check ('integer') failed - got '$value'\n";
2450     } elsif ($type eq 'number') {
2451         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2452         die "type check ('number') failed - got '$value'\n";
2453     } elsif ($type eq 'string') {
2454         if (my $fmt = $confdesc->{$key}->{format}) {
2455             PVE::JSONSchema::check_format($fmt, $value);
2456             return $value;
2457         }
2458         $value =~ s/^\"(.*)\"$/$1/;
2459         return $value;
2460     } else {
2461         die "internal error"
2462     }
2463 }
2464
2465 sub touch_config {
2466     my ($vmid) = @_;
2467
2468     my $conf = PVE::QemuConfig->config_file($vmid);
2469     utime undef, undef, $conf;
2470 }
2471
2472 sub destroy_vm {
2473     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2474
2475     my $conffile = PVE::QemuConfig->config_file($vmid);
2476
2477     my $conf = PVE::QemuConfig->load_config($vmid);
2478
2479     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2480
2481     if ($conf->{template}) {
2482         # check if any base image is still used by a linked clone
2483         foreach_drive($conf, sub {
2484                 my ($ds, $drive) = @_;
2485
2486                 return if drive_is_cdrom($drive);
2487
2488                 my $volid = $drive->{file};
2489
2490                 return if !$volid || $volid =~ m|^/|;
2491
2492                 die "base volume '$volid' is still in use by linked cloned\n"
2493                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2494
2495         });
2496     }
2497
2498     # only remove disks owned by this VM
2499     foreach_drive($conf, sub {
2500         my ($ds, $drive) = @_;
2501
2502         return if drive_is_cdrom($drive, 1);
2503
2504         my $volid = $drive->{file};
2505
2506         return if !$volid || $volid =~ m|^/|;
2507
2508         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2509         return if !$path || !$owner || ($owner != $vmid);
2510
2511         eval {
2512             PVE::Storage::vdisk_free($storecfg, $volid);
2513         };
2514         warn "Could not remove disk '$volid', check manually: $@" if $@;
2515
2516     });
2517
2518     if ($keep_empty_config) {
2519         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2520     } else {
2521         unlink $conffile;
2522     }
2523
2524     # also remove unused disk
2525     eval {
2526         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2527
2528         eval {
2529             PVE::Storage::foreach_volid($dl, sub {
2530                 my ($volid, $sid, $volname, $d) = @_;
2531                 PVE::Storage::vdisk_free($storecfg, $volid);
2532             });
2533         };
2534         warn $@ if $@;
2535
2536     };
2537     warn $@ if $@;
2538 }
2539
2540 sub parse_vm_config {
2541     my ($filename, $raw) = @_;
2542
2543     return undef if !defined($raw);
2544
2545     my $res = {
2546         digest => Digest::SHA::sha1_hex($raw),
2547         snapshots => {},
2548         pending => {},
2549     };
2550
2551     $filename =~ m|/qemu-server/(\d+)\.conf$|
2552         || die "got strange filename '$filename'";
2553
2554     my $vmid = $1;
2555
2556     my $conf = $res;
2557     my $descr;
2558     my $section = '';
2559
2560     my @lines = split(/\n/, $raw);
2561     foreach my $line (@lines) {
2562         next if $line =~ m/^\s*$/;
2563
2564         if ($line =~ m/^\[PENDING\]\s*$/i) {
2565             $section = 'pending';
2566             if (defined($descr)) {
2567                 $descr =~ s/\s+$//;
2568                 $conf->{description} = $descr;
2569             }
2570             $descr = undef;
2571             $conf = $res->{$section} = {};
2572             next;
2573
2574         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2575             $section = $1;
2576             if (defined($descr)) {
2577                 $descr =~ s/\s+$//;
2578                 $conf->{description} = $descr;
2579             }
2580             $descr = undef;
2581             $conf = $res->{snapshots}->{$section} = {};
2582             next;
2583         }
2584
2585         if ($line =~ m/^\#(.*)\s*$/) {
2586             $descr = '' if !defined($descr);
2587             $descr .= PVE::Tools::decode_text($1) . "\n";
2588             next;
2589         }
2590
2591         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2592             $descr = '' if !defined($descr);
2593             $descr .= PVE::Tools::decode_text($2);
2594         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2595             $conf->{snapstate} = $1;
2596         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2597             my $key = $1;
2598             my $value = $2;
2599             $conf->{$key} = $value;
2600         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2601             my $value = $1;
2602             if ($section eq 'pending') {
2603                 $conf->{delete} = $value; # we parse this later
2604             } else {
2605                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2606             }
2607         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2608             my $key = $1;
2609             my $value = $2;
2610             eval { $value = check_type($key, $value); };
2611             if ($@) {
2612                 warn "vm $vmid - unable to parse value of '$key' - $@";
2613             } else {
2614                 $key = 'ide2' if $key eq 'cdrom';
2615                 my $fmt = $confdesc->{$key}->{format};
2616                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2617                     my $v = parse_drive($key, $value);
2618                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2619                         $v->{file} = $volid;
2620                         $value = print_drive($vmid, $v);
2621                     } else {
2622                         warn "vm $vmid - unable to parse value of '$key'\n";
2623                         next;
2624                     }
2625                 }
2626
2627                 $conf->{$key} = $value;
2628             }
2629         }
2630     }
2631
2632     if (defined($descr)) {
2633         $descr =~ s/\s+$//;
2634         $conf->{description} = $descr;
2635     }
2636     delete $res->{snapstate}; # just to be sure
2637
2638     return $res;
2639 }
2640
2641 sub write_vm_config {
2642     my ($filename, $conf) = @_;
2643
2644     delete $conf->{snapstate}; # just to be sure
2645
2646     if ($conf->{cdrom}) {
2647         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2648         $conf->{ide2} = $conf->{cdrom};
2649         delete $conf->{cdrom};
2650     }
2651
2652     # we do not use 'smp' any longer
2653     if ($conf->{sockets}) {
2654         delete $conf->{smp};
2655     } elsif ($conf->{smp}) {
2656         $conf->{sockets} = $conf->{smp};
2657         delete $conf->{cores};
2658         delete $conf->{smp};
2659     }
2660
2661     my $used_volids = {};
2662
2663     my $cleanup_config = sub {
2664         my ($cref, $pending, $snapname) = @_;
2665
2666         foreach my $key (keys %$cref) {
2667             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2668                 $key eq 'snapstate' || $key eq 'pending';
2669             my $value = $cref->{$key};
2670             if ($key eq 'delete') {
2671                 die "propertry 'delete' is only allowed in [PENDING]\n"
2672                     if !$pending;
2673                 # fixme: check syntax?
2674                 next;
2675             }
2676             eval { $value = check_type($key, $value); };
2677             die "unable to parse value of '$key' - $@" if $@;
2678
2679             $cref->{$key} = $value;
2680
2681             if (!$snapname && is_valid_drivename($key)) {
2682                 my $drive = parse_drive($key, $value);
2683                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2684             }
2685         }
2686     };
2687
2688     &$cleanup_config($conf);
2689
2690     &$cleanup_config($conf->{pending}, 1);
2691
2692     foreach my $snapname (keys %{$conf->{snapshots}}) {
2693         die "internal error" if $snapname eq 'pending';
2694         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2695     }
2696
2697     # remove 'unusedX' settings if we re-add a volume
2698     foreach my $key (keys %$conf) {
2699         my $value = $conf->{$key};
2700         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2701             delete $conf->{$key};
2702         }
2703     }
2704
2705     my $generate_raw_config = sub {
2706         my ($conf, $pending) = @_;
2707
2708         my $raw = '';
2709
2710         # add description as comment to top of file
2711         if (defined(my $descr = $conf->{description})) {
2712             if ($descr) {
2713                 foreach my $cl (split(/\n/, $descr)) {
2714                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2715                 }
2716             } else {
2717                 $raw .= "#\n" if $pending;
2718             }
2719         }
2720
2721         foreach my $key (sort keys %$conf) {
2722             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2723             $raw .= "$key: $conf->{$key}\n";
2724         }
2725         return $raw;
2726     };
2727
2728     my $raw = &$generate_raw_config($conf);
2729
2730     if (scalar(keys %{$conf->{pending}})){
2731         $raw .= "\n[PENDING]\n";
2732         $raw .= &$generate_raw_config($conf->{pending}, 1);
2733     }
2734
2735     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2736         $raw .= "\n[$snapname]\n";
2737         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2738     }
2739
2740     return $raw;
2741 }
2742
2743 sub load_defaults {
2744
2745     my $res = {};
2746
2747     # we use static defaults from our JSON schema configuration
2748     foreach my $key (keys %$confdesc) {
2749         if (defined(my $default = $confdesc->{$key}->{default})) {
2750             $res->{$key} = $default;
2751         }
2752     }
2753
2754     return $res;
2755 }
2756
2757 sub config_list {
2758     my $vmlist = PVE::Cluster::get_vmlist();
2759     my $res = {};
2760     return $res if !$vmlist || !$vmlist->{ids};
2761     my $ids = $vmlist->{ids};
2762
2763     foreach my $vmid (keys %$ids) {
2764         my $d = $ids->{$vmid};
2765         next if !$d->{node} || $d->{node} ne $nodename;
2766         next if !$d->{type} || $d->{type} ne 'qemu';
2767         $res->{$vmid}->{exists} = 1;
2768     }
2769     return $res;
2770 }
2771
2772 # test if VM uses local resources (to prevent migration)
2773 sub check_local_resources {
2774     my ($conf, $noerr) = @_;
2775
2776     my $loc_res = 0;
2777
2778     $loc_res = 1 if $conf->{hostusb}; # old syntax
2779     $loc_res = 1 if $conf->{hostpci}; # old syntax
2780
2781     foreach my $k (keys %$conf) {
2782         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2783         # sockets are safe: they will recreated be on the target side post-migrate
2784         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2785         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2786     }
2787
2788     die "VM uses local resources\n" if $loc_res && !$noerr;
2789
2790     return $loc_res;
2791 }
2792
2793 # check if used storages are available on all nodes (use by migrate)
2794 sub check_storage_availability {
2795     my ($storecfg, $conf, $node) = @_;
2796
2797     foreach_drive($conf, sub {
2798         my ($ds, $drive) = @_;
2799
2800         my $volid = $drive->{file};
2801         return if !$volid;
2802
2803         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2804         return if !$sid;
2805
2806         # check if storage is available on both nodes
2807         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2808         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2809    });
2810 }
2811
2812 # list nodes where all VM images are available (used by has_feature API)
2813 sub shared_nodes {
2814     my ($conf, $storecfg) = @_;
2815
2816     my $nodelist = PVE::Cluster::get_nodelist();
2817     my $nodehash = { map { $_ => 1 } @$nodelist };
2818     my $nodename = PVE::INotify::nodename();
2819
2820     foreach_drive($conf, sub {
2821         my ($ds, $drive) = @_;
2822
2823         my $volid = $drive->{file};
2824         return if !$volid;
2825
2826         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2827         if ($storeid) {
2828             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2829             if ($scfg->{disable}) {
2830                 $nodehash = {};
2831             } elsif (my $avail = $scfg->{nodes}) {
2832                 foreach my $node (keys %$nodehash) {
2833                     delete $nodehash->{$node} if !$avail->{$node};
2834                 }
2835             } elsif (!$scfg->{shared}) {
2836                 foreach my $node (keys %$nodehash) {
2837                     delete $nodehash->{$node} if $node ne $nodename
2838                 }
2839             }
2840         }
2841     });
2842
2843     return $nodehash
2844 }
2845
2846 sub check_cmdline {
2847     my ($pidfile, $pid) = @_;
2848
2849     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2850     if (defined($fh)) {
2851         my $line = <$fh>;
2852         $fh->close;
2853         return undef if !$line;
2854         my @param = split(/\0/, $line);
2855
2856         my $cmd = $param[0];
2857         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2858
2859         for (my $i = 0; $i < scalar (@param); $i++) {
2860             my $p = $param[$i];
2861             next if !$p;
2862             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2863                 my $p = $param[$i+1];
2864                 return 1 if $p && ($p eq $pidfile);
2865                 return undef;
2866             }
2867         }
2868     }
2869     return undef;
2870 }
2871
2872 sub check_running {
2873     my ($vmid, $nocheck, $node) = @_;
2874
2875     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2876
2877     die "unable to find configuration file for VM $vmid - no such machine\n"
2878         if !$nocheck && ! -f $filename;
2879
2880     my $pidfile = pidfile_name($vmid);
2881
2882     if (my $fd = IO::File->new("<$pidfile")) {
2883         my $st = stat($fd);
2884         my $line = <$fd>;
2885         close($fd);
2886
2887         my $mtime = $st->mtime;
2888         if ($mtime > time()) {
2889             warn "file '$filename' modified in future\n";
2890         }
2891
2892         if ($line =~ m/^(\d+)$/) {
2893             my $pid = $1;
2894             if (check_cmdline($pidfile, $pid)) {
2895                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2896                     return $pid;
2897                 }
2898             }
2899         }
2900     }
2901
2902     return undef;
2903 }
2904
2905 sub vzlist {
2906
2907     my $vzlist = config_list();
2908
2909     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2910
2911     while (defined(my $de = $fd->read)) {
2912         next if $de !~ m/^(\d+)\.pid$/;
2913         my $vmid = $1;
2914         next if !defined($vzlist->{$vmid});
2915         if (my $pid = check_running($vmid)) {
2916             $vzlist->{$vmid}->{pid} = $pid;
2917         }
2918     }
2919
2920     return $vzlist;
2921 }
2922
2923 sub disksize {
2924     my ($storecfg, $conf) = @_;
2925
2926     my $bootdisk = $conf->{bootdisk};
2927     return undef if !$bootdisk;
2928     return undef if !is_valid_drivename($bootdisk);
2929
2930     return undef if !$conf->{$bootdisk};
2931
2932     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2933     return undef if !defined($drive);
2934
2935     return undef if drive_is_cdrom($drive);
2936
2937     my $volid = $drive->{file};
2938     return undef if !$volid;
2939
2940     return $drive->{size};
2941 }
2942
2943 our $vmstatus_return_properties = {
2944     vmid => get_standard_option('pve-vmid'),
2945     status => {
2946         description => "Qemu process status.",
2947         type => 'string',
2948         enum => ['stopped', 'running'],
2949     },
2950     maxmem => {
2951         description => "Maximum memory in bytes.",
2952         type => 'integer',
2953         optional => 1,
2954         renderer => 'bytes',
2955     },
2956     maxdisk => {
2957         description => "Root disk size in bytes.",
2958         type => 'integer',
2959         optional => 1,
2960         renderer => 'bytes',
2961     },
2962     name => {
2963         description => "VM name.",
2964         type => 'string',
2965         optional => 1,
2966     },
2967     qmpstatus => {
2968         description => "Qemu QMP agent status.",
2969         type => 'string',
2970         optional => 1,
2971     },
2972     pid => {
2973         description => "PID of running qemu process.",
2974         type => 'integer',
2975         optional => 1,
2976     },
2977     uptime => {
2978         description => "Uptime.",
2979         type => 'integer',
2980         optional => 1,
2981         renderer => 'duration',
2982     },
2983     cpus => {
2984         description => "Maximum usable CPUs.",
2985         type => 'number',
2986         optional => 1,
2987     },
2988 };
2989
2990 my $last_proc_pid_stat;
2991
2992 # get VM status information
2993 # This must be fast and should not block ($full == false)
2994 # We only query KVM using QMP if $full == true (this can be slow)
2995 sub vmstatus {
2996     my ($opt_vmid, $full) = @_;
2997
2998     my $res = {};
2999
3000     my $storecfg = PVE::Storage::config();
3001
3002     my $list = vzlist();
3003     my $defaults = load_defaults();
3004
3005     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3006
3007     my $cpucount = $cpuinfo->{cpus} || 1;
3008
3009     foreach my $vmid (keys %$list) {
3010         next if $opt_vmid && ($vmid ne $opt_vmid);
3011
3012         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3013         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3014
3015         my $d = { vmid => $vmid };
3016         $d->{pid} = $list->{$vmid}->{pid};
3017
3018         # fixme: better status?
3019         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3020
3021         my $size = disksize($storecfg, $conf);
3022         if (defined($size)) {
3023             $d->{disk} = 0; # no info available
3024             $d->{maxdisk} = $size;
3025         } else {
3026             $d->{disk} = 0;
3027             $d->{maxdisk} = 0;
3028         }
3029
3030         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3031             * ($conf->{cores} || $defaults->{cores});
3032         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3033         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3034
3035         $d->{name} = $conf->{name} || "VM $vmid";
3036         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3037             : $defaults->{memory}*(1024*1024);
3038
3039         if ($conf->{balloon}) {
3040             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3041             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3042                 : $defaults->{shares};
3043         }
3044
3045         $d->{uptime} = 0;
3046         $d->{cpu} = 0;
3047         $d->{mem} = 0;
3048
3049         $d->{netout} = 0;
3050         $d->{netin} = 0;
3051
3052         $d->{diskread} = 0;
3053         $d->{diskwrite} = 0;
3054
3055         $d->{template} = PVE::QemuConfig->is_template($conf);
3056
3057         $d->{serial} = 1 if conf_has_serial($conf);
3058
3059         $res->{$vmid} = $d;
3060     }
3061
3062     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3063     foreach my $dev (keys %$netdev) {
3064         next if $dev !~ m/^tap([1-9]\d*)i/;
3065         my $vmid = $1;
3066         my $d = $res->{$vmid};
3067         next if !$d;
3068
3069         $d->{netout} += $netdev->{$dev}->{receive};
3070         $d->{netin} += $netdev->{$dev}->{transmit};
3071
3072         if ($full) {
3073             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3074             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3075         }
3076
3077     }
3078
3079     my $ctime = gettimeofday;
3080
3081     foreach my $vmid (keys %$list) {
3082
3083         my $d = $res->{$vmid};
3084         my $pid = $d->{pid};
3085         next if !$pid;
3086
3087         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3088         next if !$pstat; # not running
3089
3090         my $used = $pstat->{utime} + $pstat->{stime};
3091
3092         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3093
3094         if ($pstat->{vsize}) {
3095             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3096         }
3097
3098         my $old = $last_proc_pid_stat->{$pid};
3099         if (!$old) {
3100             $last_proc_pid_stat->{$pid} = {
3101                 time => $ctime,
3102                 used => $used,
3103                 cpu => 0,
3104             };
3105             next;
3106         }
3107
3108         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3109
3110         if ($dtime > 1000) {
3111             my $dutime = $used -  $old->{used};
3112
3113             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3114             $last_proc_pid_stat->{$pid} = {
3115                 time => $ctime,
3116                 used => $used,
3117                 cpu => $d->{cpu},
3118             };
3119         } else {
3120             $d->{cpu} = $old->{cpu};
3121         }
3122     }
3123
3124     return $res if !$full;
3125
3126     my $qmpclient = PVE::QMPClient->new();
3127
3128     my $ballooncb = sub {
3129         my ($vmid, $resp) = @_;
3130
3131         my $info = $resp->{'return'};
3132         return if !$info->{max_mem};
3133
3134         my $d = $res->{$vmid};
3135
3136         # use memory assigned to VM
3137         $d->{maxmem} = $info->{max_mem};
3138         $d->{balloon} = $info->{actual};
3139
3140         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3141             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3142             $d->{freemem} = $info->{free_mem};
3143         }
3144
3145         $d->{ballooninfo} = $info;
3146     };
3147
3148     my $blockstatscb = sub {
3149         my ($vmid, $resp) = @_;
3150         my $data = $resp->{'return'} || [];
3151         my $totalrdbytes = 0;
3152         my $totalwrbytes = 0;
3153
3154         for my $blockstat (@$data) {
3155             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3156             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3157
3158             $blockstat->{device} =~ s/drive-//;
3159             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3160         }
3161         $res->{$vmid}->{diskread} = $totalrdbytes;
3162         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3163     };
3164
3165     my $statuscb = sub {
3166         my ($vmid, $resp) = @_;
3167
3168         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3169         # this fails if ballon driver is not loaded, so this must be
3170         # the last commnand (following command are aborted if this fails).
3171         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3172
3173         my $status = 'unknown';
3174         if (!defined($status = $resp->{'return'}->{status})) {
3175             warn "unable to get VM status\n";
3176             return;
3177         }
3178
3179         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3180     };
3181
3182     foreach my $vmid (keys %$list) {
3183         next if $opt_vmid && ($vmid ne $opt_vmid);
3184         next if !$res->{$vmid}->{pid}; # not running
3185         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3186     }
3187
3188     $qmpclient->queue_execute(undef, 2);
3189
3190     foreach my $vmid (keys %$list) {
3191         next if $opt_vmid && ($vmid ne $opt_vmid);
3192         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3193     }
3194
3195     return $res;
3196 }
3197
3198 sub foreach_drive {
3199     my ($conf, $func, @param) = @_;
3200
3201     foreach my $ds (valid_drive_names()) {
3202         next if !defined($conf->{$ds});
3203
3204         my $drive = parse_drive($ds, $conf->{$ds});
3205         next if !$drive;
3206
3207         &$func($ds, $drive, @param);
3208     }
3209 }
3210
3211 sub foreach_volid {
3212     my ($conf, $func, @param) = @_;
3213
3214     my $volhash = {};
3215
3216     my $test_volid = sub {
3217         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3218
3219         return if !$volid;
3220
3221         $volhash->{$volid}->{cdrom} //= 1;
3222         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3223
3224         $volhash->{$volid}->{replicate} //= 0;
3225         $volhash->{$volid}->{replicate} = 1 if $replicate;
3226
3227         $volhash->{$volid}->{shared} //= 0;
3228         $volhash->{$volid}->{shared} = 1 if $shared;
3229
3230         $volhash->{$volid}->{referenced_in_config} //= 0;
3231         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3232
3233         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3234             if defined($snapname);
3235     };
3236
3237     foreach_drive($conf, sub {
3238         my ($ds, $drive) = @_;
3239         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3240     });
3241
3242     foreach my $snapname (keys %{$conf->{snapshots}}) {
3243         my $snap = $conf->{snapshots}->{$snapname};
3244         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3245         foreach_drive($snap, sub {
3246             my ($ds, $drive) = @_;
3247             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3248         });
3249     }
3250
3251     foreach my $volid (keys %$volhash) {
3252         &$func($volid, $volhash->{$volid}, @param);
3253     }
3254 }
3255
3256 sub conf_has_serial {
3257     my ($conf) = @_;
3258
3259     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3260         if ($conf->{"serial$i"}) {
3261             return 1;
3262         }
3263     }
3264
3265     return 0;
3266 }
3267
3268 sub vga_conf_has_spice {
3269     my ($vga) = @_;
3270
3271     my $vgaconf = parse_vga($vga);
3272     my $vgatype = $vgaconf->{type};
3273     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3274
3275     return $1 || 1;
3276 }
3277
3278 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3279 sub get_host_arch() {
3280     $host_arch = (POSIX::uname())[4] if !$host_arch;
3281     return $host_arch;
3282 }
3283
3284 sub is_native($) {
3285     my ($arch) = @_;
3286     return get_host_arch() eq $arch;
3287 }
3288
3289 my $default_machines = {
3290     x86_64 => 'pc',
3291     aarch64 => 'virt',
3292 };
3293
3294 sub get_basic_machine_info {
3295     my ($conf, $forcemachine) = @_;
3296
3297     my $arch = $conf->{arch} // get_host_arch();
3298     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3299     return ($arch, $machine);
3300 }
3301
3302 sub get_ovmf_files($) {
3303     my ($arch) = @_;
3304
3305     my $ovmf = $OVMF->{$arch}
3306         or die "no OVMF images known for architecture '$arch'\n";
3307
3308     return @$ovmf;
3309 }
3310
3311 my $Arch2Qemu = {
3312     aarch64 => '/usr/bin/qemu-system-aarch64',
3313     x86_64 => '/usr/bin/qemu-system-x86_64',
3314 };
3315 sub get_command_for_arch($) {
3316     my ($arch) = @_;
3317     return '/usr/bin/kvm' if is_native($arch);
3318
3319     my $cmd = $Arch2Qemu->{$arch}
3320         or die "don't know how to emulate architecture '$arch'\n";
3321     return $cmd;
3322 }
3323
3324 sub get_cpu_options {
3325     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3326
3327     my $cpuFlags = [];
3328     my $ostype = $conf->{ostype};
3329
3330     my $cpu = $kvm ? "kvm64" : "qemu64";
3331     if ($arch eq 'aarch64') {
3332         $cpu = 'cortex-a57';
3333     }
3334     if (my $cputype = $conf->{cpu}) {
3335         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3336             or die "Cannot parse cpu description: $cputype\n";
3337         $cpu = $cpuconf->{cputype};
3338         $kvm_off = 1 if $cpuconf->{hidden};
3339
3340         if (defined(my $flags = $cpuconf->{flags})) {
3341             push @$cpuFlags, split(";", $flags);
3342         }
3343     }
3344
3345     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3346
3347     push @$cpuFlags , '-x2apic'
3348         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3349
3350     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3351
3352     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3353
3354     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3355
3356         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3357         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3358     }
3359
3360     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3361
3362     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3363
3364     push @$cpuFlags, 'kvm=off' if $kvm_off;
3365
3366     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3367         push @$cpuFlags, "vendor=${cpu_vendor}"
3368             if $cpu_vendor ne 'default';
3369     } elsif ($arch ne 'aarch64') {
3370         die "internal error"; # should not happen
3371     }
3372
3373     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3374
3375     return ('-cpu', $cpu);
3376 }
3377
3378 sub config_to_command {
3379     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3380
3381     my $cmd = [];
3382     my $globalFlags = [];
3383     my $machineFlags = [];
3384     my $rtcFlags = [];
3385     my $devices = [];
3386     my $pciaddr = '';
3387     my $bridges = {};
3388     my $kvmver = kvm_user_version();
3389     my $vernum = 0; # unknown
3390     my $ostype = $conf->{ostype};
3391     my $winversion = windows_version($ostype);
3392     my $kvm = $conf->{kvm};
3393
3394     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3395     $kvm //= 1 if is_native($arch);
3396
3397     if ($kvm) {
3398         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3399             if !defined kvm_version();
3400     }
3401
3402     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3403         $vernum = $1*1000000+$2*1000;
3404     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3405         $vernum = $1*1000000+$2*1000+$3;
3406     }
3407
3408     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3409
3410     my $have_ovz = -f '/proc/vz/vestat';
3411
3412     my $q35 = machine_type_is_q35($conf);
3413     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3414     my $use_old_bios_files = undef;
3415     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3416
3417     my $cpuunits = defined($conf->{cpuunits}) ?
3418             $conf->{cpuunits} : $defaults->{cpuunits};
3419
3420     push @$cmd, get_command_for_arch($arch);
3421
3422     push @$cmd, '-id', $vmid;
3423
3424     my $vmname = $conf->{name} || "vm$vmid";
3425
3426     push @$cmd, '-name', $vmname;
3427
3428     my $use_virtio = 0;
3429
3430     my $qmpsocket = qmp_socket($vmid);
3431     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3432     push @$cmd, '-mon', "chardev=qmp,mode=control";
3433
3434     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3435         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3436         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3437     }
3438
3439     push @$cmd, '-pidfile' , pidfile_name($vmid);
3440
3441     push @$cmd, '-daemonize';
3442
3443     if ($conf->{smbios1}) {
3444         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3445     }
3446
3447     if ($conf->{vmgenid}) {
3448         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3449     }
3450
3451     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3452     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3453         die "uefi base image not found\n" if ! -f $ovmf_code;
3454
3455         my $path;
3456         my $format;
3457         if (my $efidisk = $conf->{efidisk0}) {
3458             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3459             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3460             $format = $d->{format};
3461             if ($storeid) {
3462                 $path = PVE::Storage::path($storecfg, $d->{file});
3463                 if (!defined($format)) {
3464                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3465                     $format = qemu_img_format($scfg, $volname);
3466                 }
3467             } else {
3468                 $path = $d->{file};
3469                 die "efidisk format must be specified\n"
3470                     if !defined($format);
3471             }
3472         } else {
3473             warn "no efidisk configured! Using temporary efivars disk.\n";
3474             $path = "/tmp/$vmid-ovmf.fd";
3475             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3476             $format = 'raw';
3477         }
3478
3479         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3480         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3481     }
3482
3483
3484     # add usb controllers
3485     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3486     push @$devices, @usbcontrollers if @usbcontrollers;
3487     my $vga = parse_vga($conf->{vga});
3488
3489     my $qxlnum = vga_conf_has_spice($conf->{vga});
3490     $vga->{type} = 'qxl' if $qxlnum;
3491
3492     if (!$vga->{type}) {
3493         if ($arch eq 'aarch64') {
3494             $vga->{type} = 'virtio';
3495         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3496             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3497         } else {
3498             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3499         }
3500     }
3501
3502     # enable absolute mouse coordinates (needed by vnc)
3503     my $tablet;
3504     if (defined($conf->{tablet})) {
3505         $tablet = $conf->{tablet};
3506     } else {
3507         $tablet = $defaults->{tablet};
3508         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3509         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3510     }
3511
3512     if ($tablet) {
3513         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3514         my $kbd = print_keyboarddevice_full($conf, $arch);
3515         push @$devices, '-device', $kbd if defined($kbd);
3516     }
3517
3518     my $kvm_off = 0;
3519     my $gpu_passthrough;
3520
3521     # host pci devices
3522     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3523         my $d = parse_hostpci($conf->{"hostpci$i"});
3524         next if !$d;
3525
3526         my $pcie = $d->{pcie};
3527         if($pcie){
3528             die "q35 machine model is not enabled" if !$q35;
3529             # win7 wants to have the pcie devices directly on the pcie bus
3530             # instead of in the root port
3531             if ($winversion == 7) {
3532                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3533             } else {
3534                 $pciaddr = print_pcie_addr("hostpci$i");
3535             }
3536         }else{
3537             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3538         }
3539
3540         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3541         my $romfile = $d->{romfile};
3542
3543         my $xvga = '';
3544         if ($d->{'x-vga'}) {
3545             $xvga = ',x-vga=on';
3546             $kvm_off = 1;
3547             $vga->{type} = 'none';
3548             $gpu_passthrough = 1;
3549
3550             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3551                 $xvga = "";
3552             }
3553         }
3554         my $pcidevices = $d->{pciid};
3555         my $multifunction = 1 if @$pcidevices > 1;
3556         my $sysfspath;
3557         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3558             my $id = $pcidevices->[0]->{id};
3559             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3560             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3561         } elsif ($d->{mdev}) {
3562             warn "ignoring mediated device with multifunction device\n";
3563         }
3564
3565         my $j=0;
3566         foreach my $pcidevice (@$pcidevices) {
3567
3568             my $id = "hostpci$i";
3569             $id .= ".$j" if $multifunction;
3570             my $addr = $pciaddr;
3571             $addr .= ".$j" if $multifunction;
3572             my $devicestr = "vfio-pci";
3573             if ($sysfspath) {
3574                 $devicestr .= ",sysfsdev=$sysfspath";
3575             } else {
3576                 $devicestr .= ",host=$pcidevice->{id}";
3577             }
3578             $devicestr .= ",id=$id$addr";
3579
3580             if($j == 0){
3581                 $devicestr .= "$rombar$xvga";
3582                 $devicestr .= ",multifunction=on" if $multifunction;
3583                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3584             }
3585
3586             push @$devices, '-device', $devicestr;
3587             $j++;
3588         }
3589     }
3590
3591     # usb devices
3592     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3593     push @$devices, @usbdevices if @usbdevices;
3594     # serial devices
3595     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3596         if (my $path = $conf->{"serial$i"}) {
3597             if ($path eq 'socket') {
3598                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3599                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3600                 # On aarch64, serial0 is the UART device. Qemu only allows
3601                 # connecting UART devices via the '-serial' command line, as
3602                 # the device has a fixed slot on the hardware...
3603                 if ($arch eq 'aarch64' && $i == 0) {
3604                     push @$devices, '-serial', "chardev:serial$i";
3605                 } else {
3606                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3607                 }
3608             } else {
3609                 die "no such serial device\n" if ! -c $path;
3610                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3611                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3612             }
3613         }
3614     }
3615
3616     # parallel devices
3617     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3618         if (my $path = $conf->{"parallel$i"}) {
3619             die "no such parallel device\n" if ! -c $path;
3620             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3621             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3622             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3623         }
3624     }
3625
3626
3627     my $sockets = 1;
3628     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3629     $sockets = $conf->{sockets} if  $conf->{sockets};
3630
3631     my $cores = $conf->{cores} || 1;
3632
3633     my $maxcpus = $sockets * $cores;
3634
3635     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3636
3637     my $allowed_vcpus = $cpuinfo->{cpus};
3638
3639     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3640         if ($allowed_vcpus < $maxcpus);
3641
3642     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3643
3644         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3645         for (my $i = 2; $i <= $vcpus; $i++)  {
3646             my $cpustr = print_cpu_device($conf,$i);
3647             push @$cmd, '-device', $cpustr;
3648         }
3649
3650     } else {
3651
3652         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3653     }
3654     push @$cmd, '-nodefaults';
3655
3656     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3657
3658     my $bootindex_hash = {};
3659     my $i = 1;
3660     foreach my $o (split(//, $bootorder)) {
3661         $bootindex_hash->{$o} = $i*100;
3662         $i++;
3663     }
3664
3665     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3666
3667     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3668
3669     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3670
3671     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3672         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3673         my $socket = vnc_socket($vmid);
3674         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3675     } else {
3676         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3677         push @$cmd, '-nographic';
3678     }
3679
3680     # time drift fix
3681     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3682
3683     my $useLocaltime = $conf->{localtime};
3684
3685     if ($winversion >= 5) { # windows
3686         $useLocaltime = 1 if !defined($conf->{localtime});
3687
3688         # use time drift fix when acpi is enabled
3689         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3690             $tdf = 1 if !defined($conf->{tdf});
3691         }
3692     }
3693
3694     if ($winversion >= 6) {
3695         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3696         push @$cmd, '-no-hpet';
3697     }
3698
3699     push @$rtcFlags, 'driftfix=slew' if $tdf;
3700
3701     if (!$kvm) {
3702         push @$machineFlags, 'accel=tcg';
3703     }
3704
3705     if ($machine_type) {
3706         push @$machineFlags, "type=${machine_type}";
3707     }
3708
3709     if ($conf->{startdate}) {
3710         push @$rtcFlags, "base=$conf->{startdate}";
3711     } elsif ($useLocaltime) {
3712         push @$rtcFlags, 'base=localtime';
3713     }
3714
3715     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3716
3717     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3718
3719     push @$cmd, '-S' if $conf->{freeze};
3720
3721     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3722
3723     # enable sound
3724     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3725     #push @$cmd, '-soundhw', 'es1370';
3726     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3727
3728     if (parse_guest_agent($conf)->{enabled}) {
3729         my $qgasocket = qmp_socket($vmid, 1);
3730         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3731         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3732         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3733         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3734     }
3735
3736     my $spice_port;
3737
3738     if ($qxlnum) {
3739         if ($qxlnum > 1) {
3740             if ($winversion){
3741                 for(my $i = 1; $i < $qxlnum; $i++){
3742                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3743                 }
3744             } else {
3745                 # assume other OS works like Linux
3746                 my ($ram, $vram) = ("134217728", "67108864");
3747                 if ($vga->{memory}) {
3748                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3749                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3750                 }
3751                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3752                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3753             }
3754         }
3755
3756         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3757
3758         my $nodename = PVE::INotify::nodename();
3759         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3760         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3761         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3762         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3763         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3764
3765         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3766
3767         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3768         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3769         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3770     }
3771
3772     # enable balloon by default, unless explicitly disabled
3773     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3774         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3775         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3776     }
3777
3778     if ($conf->{watchdog}) {
3779         my $wdopts = parse_watchdog($conf->{watchdog});
3780         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3781         my $watchdog = $wdopts->{model} || 'i6300esb';
3782         push @$devices, '-device', "$watchdog$pciaddr";
3783         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3784     }
3785
3786     my $vollist = [];
3787     my $scsicontroller = {};
3788     my $ahcicontroller = {};
3789     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3790
3791     # Add iscsi initiator name if available
3792     if (my $initiator = get_initiator_name()) {
3793         push @$devices, '-iscsi', "initiator-name=$initiator";
3794     }
3795
3796     foreach_drive($conf, sub {
3797         my ($ds, $drive) = @_;
3798
3799         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3800             push @$vollist, $drive->{file};
3801         }
3802
3803         # ignore efidisk here, already added in bios/fw handling code above
3804         return if $drive->{interface} eq 'efidisk';
3805
3806         $use_virtio = 1 if $ds =~ m/^virtio/;
3807
3808         if (drive_is_cdrom ($drive)) {
3809             if ($bootindex_hash->{d}) {
3810                 $drive->{bootindex} = $bootindex_hash->{d};
3811                 $bootindex_hash->{d} += 1;
3812             }
3813         } else {
3814             if ($bootindex_hash->{c}) {
3815                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3816                 $bootindex_hash->{c} += 1;
3817             }
3818         }
3819
3820         if($drive->{interface} eq 'virtio'){
3821            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3822         }
3823
3824         if ($drive->{interface} eq 'scsi') {
3825
3826             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3827
3828             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3829             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3830
3831             my $iothread = '';
3832             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3833                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3834                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3835             } elsif ($drive->{iothread}) {
3836                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3837             }
3838
3839             my $queues = '';
3840             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3841                 $queues = ",num_queues=$drive->{queues}";
3842             }
3843
3844             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3845             $scsicontroller->{$controller}=1;
3846         }
3847
3848         if ($drive->{interface} eq 'sata') {
3849            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3850            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3851            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3852            $ahcicontroller->{$controller}=1;
3853         }
3854
3855         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3856         push @$devices, '-drive',$drive_cmd;
3857         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3858     });
3859
3860     for (my $i = 0; $i < $MAX_NETS; $i++) {
3861          next if !$conf->{"net$i"};
3862          my $d = parse_net($conf->{"net$i"});
3863          next if !$d;
3864
3865          $use_virtio = 1 if $d->{model} eq 'virtio';
3866
3867          if ($bootindex_hash->{n}) {
3868             $d->{bootindex} = $bootindex_hash->{n};
3869             $bootindex_hash->{n} += 1;
3870          }
3871
3872          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3873          push @$devices, '-netdev', $netdevfull;
3874
3875          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3876          push @$devices, '-device', $netdevicefull;
3877     }
3878
3879     if (!$q35) {
3880         # add pci bridges
3881         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3882            $bridges->{1} = 1;
3883            $bridges->{2} = 1;
3884         }
3885
3886         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3887
3888         while (my ($k, $v) = each %$bridges) {
3889             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3890             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3891         }
3892     }
3893
3894     push @$cmd, @$devices;
3895     push @$cmd, '-rtc', join(',', @$rtcFlags)
3896         if scalar(@$rtcFlags);
3897     push @$cmd, '-machine', join(',', @$machineFlags)
3898         if scalar(@$machineFlags);
3899     push @$cmd, '-global', join(',', @$globalFlags)
3900         if scalar(@$globalFlags);
3901
3902     # add custom args
3903     if ($conf->{args}) {
3904         my $aa = PVE::Tools::split_args($conf->{args});
3905         push @$cmd, @$aa;
3906     }
3907
3908     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3909 }
3910
3911 sub vnc_socket {
3912     my ($vmid) = @_;
3913     return "${var_run_tmpdir}/$vmid.vnc";
3914 }
3915
3916 sub spice_port {
3917     my ($vmid) = @_;
3918
3919     my $res = vm_mon_cmd($vmid, 'query-spice');
3920
3921     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3922 }
3923
3924 sub qmp_socket {
3925     my ($vmid, $qga, $name) = @_;
3926     my $sockettype = $qga ? 'qga' : 'qmp';
3927     my $ext = $name ? '-'.$name : '';
3928     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3929 }
3930
3931 sub pidfile_name {
3932     my ($vmid) = @_;
3933     return "${var_run_tmpdir}/$vmid.pid";
3934 }
3935
3936 sub vm_devices_list {
3937     my ($vmid) = @_;
3938
3939     my $res = vm_mon_cmd($vmid, 'query-pci');
3940     my $devices_to_check = [];
3941     my $devices = {};
3942     foreach my $pcibus (@$res) {
3943         push @$devices_to_check, @{$pcibus->{devices}},
3944     }
3945
3946     while (@$devices_to_check) {
3947         my $to_check = [];
3948         for my $d (@$devices_to_check) {
3949             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3950             next if !$d->{'pci_bridge'};
3951
3952             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3953             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3954         }
3955         $devices_to_check = $to_check;
3956     }
3957
3958     my $resblock = vm_mon_cmd($vmid, 'query-block');
3959     foreach my $block (@$resblock) {
3960         if($block->{device} =~ m/^drive-(\S+)/){
3961                 $devices->{$1} = 1;
3962         }
3963     }
3964
3965     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3966     foreach my $mice (@$resmice) {
3967         if ($mice->{name} eq 'QEMU HID Tablet') {
3968             $devices->{tablet} = 1;
3969             last;
3970         }
3971     }
3972
3973     # for usb devices there is no query-usb
3974     # but we can iterate over the entries in
3975     # qom-list path=/machine/peripheral
3976     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3977     foreach my $per (@$resperipheral) {
3978         if ($per->{name} =~ m/^usb\d+$/) {
3979             $devices->{$per->{name}} = 1;
3980         }
3981     }
3982
3983     return $devices;
3984 }
3985
3986 sub vm_deviceplug {
3987     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3988
3989     my $q35 = machine_type_is_q35($conf);
3990
3991     my $devices_list = vm_devices_list($vmid);
3992     return 1 if defined($devices_list->{$deviceid});
3993
3994     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3995
3996     if ($deviceid eq 'tablet') {
3997
3998         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3999
4000     } elsif ($deviceid eq 'keyboard') {
4001
4002         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4003
4004     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4005
4006         die "usb hotplug currently not reliable\n";
4007         # since we can't reliably hot unplug all added usb devices
4008         # and usb passthrough disables live migration
4009         # we disable usb hotplugging for now
4010         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4011
4012     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4013
4014         qemu_iothread_add($vmid, $deviceid, $device);
4015
4016         qemu_driveadd($storecfg, $vmid, $device);
4017         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4018
4019         qemu_deviceadd($vmid, $devicefull);
4020         eval { qemu_deviceaddverify($vmid, $deviceid); };
4021         if (my $err = $@) {
4022             eval { qemu_drivedel($vmid, $deviceid); };
4023             warn $@ if $@;
4024             die $err;
4025         }
4026
4027     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4028
4029
4030         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4031         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4032         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4033
4034         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4035
4036         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4037             qemu_iothread_add($vmid, $deviceid, $device);
4038             $devicefull .= ",iothread=iothread-$deviceid";
4039         }
4040
4041         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4042             $devicefull .= ",num_queues=$device->{queues}";
4043         }
4044
4045         qemu_deviceadd($vmid, $devicefull);
4046         qemu_deviceaddverify($vmid, $deviceid);
4047
4048     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4049
4050         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4051         qemu_driveadd($storecfg, $vmid, $device);
4052
4053         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4054         eval { qemu_deviceadd($vmid, $devicefull); };
4055         if (my $err = $@) {
4056             eval { qemu_drivedel($vmid, $deviceid); };
4057             warn $@ if $@;
4058             die $err;
4059         }
4060
4061     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4062
4063         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4064
4065         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4066         my $use_old_bios_files = undef;
4067         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4068
4069         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4070         qemu_deviceadd($vmid, $netdevicefull);
4071         eval { qemu_deviceaddverify($vmid, $deviceid); };
4072         if (my $err = $@) {
4073             eval { qemu_netdevdel($vmid, $deviceid); };
4074             warn $@ if $@;
4075             die $err;
4076         }
4077
4078     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4079
4080         my $bridgeid = $2;
4081         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4082         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4083
4084         qemu_deviceadd($vmid, $devicefull);
4085         qemu_deviceaddverify($vmid, $deviceid);
4086
4087     } else {
4088         die "can't hotplug device '$deviceid'\n";
4089     }
4090
4091     return 1;
4092 }
4093
4094 # fixme: this should raise exceptions on error!
4095 sub vm_deviceunplug {
4096     my ($vmid, $conf, $deviceid) = @_;
4097
4098     my $devices_list = vm_devices_list($vmid);
4099     return 1 if !defined($devices_list->{$deviceid});
4100
4101     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4102
4103     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4104
4105         qemu_devicedel($vmid, $deviceid);
4106
4107     } elsif ($deviceid =~ m/^usb\d+$/) {
4108
4109         die "usb hotplug currently not reliable\n";
4110         # when unplugging usb devices this way,
4111         # there may be remaining usb controllers/hubs
4112         # so we disable it for now
4113         qemu_devicedel($vmid, $deviceid);
4114         qemu_devicedelverify($vmid, $deviceid);
4115
4116     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4117
4118         qemu_devicedel($vmid, $deviceid);
4119         qemu_devicedelverify($vmid, $deviceid);
4120         qemu_drivedel($vmid, $deviceid);
4121         qemu_iothread_del($conf, $vmid, $deviceid);
4122
4123     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4124
4125         qemu_devicedel($vmid, $deviceid);
4126         qemu_devicedelverify($vmid, $deviceid);
4127         qemu_iothread_del($conf, $vmid, $deviceid);
4128
4129     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4130
4131         qemu_devicedel($vmid, $deviceid);
4132         qemu_drivedel($vmid, $deviceid);
4133         qemu_deletescsihw($conf, $vmid, $deviceid);
4134
4135     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4136
4137         qemu_devicedel($vmid, $deviceid);
4138         qemu_devicedelverify($vmid, $deviceid);
4139         qemu_netdevdel($vmid, $deviceid);
4140
4141     } else {
4142         die "can't unplug device '$deviceid'\n";
4143     }
4144
4145     return 1;
4146 }
4147
4148 sub qemu_deviceadd {
4149     my ($vmid, $devicefull) = @_;
4150
4151     $devicefull = "driver=".$devicefull;
4152     my %options =  split(/[=,]/, $devicefull);
4153
4154     vm_mon_cmd($vmid, "device_add" , %options);
4155 }
4156
4157 sub qemu_devicedel {
4158     my ($vmid, $deviceid) = @_;
4159
4160     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4161 }
4162
4163 sub qemu_iothread_add {
4164     my($vmid, $deviceid, $device) = @_;
4165
4166     if ($device->{iothread}) {
4167         my $iothreads = vm_iothreads_list($vmid);
4168         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4169     }
4170 }
4171
4172 sub qemu_iothread_del {
4173     my($conf, $vmid, $deviceid) = @_;
4174
4175     my $device = parse_drive($deviceid, $conf->{$deviceid});
4176     if ($device->{iothread}) {
4177         my $iothreads = vm_iothreads_list($vmid);
4178         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4179     }
4180 }
4181
4182 sub qemu_objectadd {
4183     my($vmid, $objectid, $qomtype) = @_;
4184
4185     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4186
4187     return 1;
4188 }
4189
4190 sub qemu_objectdel {
4191     my($vmid, $objectid) = @_;
4192
4193     vm_mon_cmd($vmid, "object-del", id => $objectid);
4194
4195     return 1;
4196 }
4197
4198 sub qemu_driveadd {
4199     my ($storecfg, $vmid, $device) = @_;
4200
4201     my $drive = print_drive_full($storecfg, $vmid, $device);
4202     $drive =~ s/\\/\\\\/g;
4203     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4204
4205     # If the command succeeds qemu prints: "OK"
4206     return 1 if $ret =~ m/OK/s;
4207
4208     die "adding drive failed: $ret\n";
4209 }
4210
4211 sub qemu_drivedel {
4212     my($vmid, $deviceid) = @_;
4213
4214     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4215     $ret =~ s/^\s+//;
4216
4217     return 1 if $ret eq "";
4218
4219     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4220     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4221
4222     die "deleting drive $deviceid failed : $ret\n";
4223 }
4224
4225 sub qemu_deviceaddverify {
4226     my ($vmid, $deviceid) = @_;
4227
4228     for (my $i = 0; $i <= 5; $i++) {
4229          my $devices_list = vm_devices_list($vmid);
4230          return 1 if defined($devices_list->{$deviceid});
4231          sleep 1;
4232     }
4233
4234     die "error on hotplug device '$deviceid'\n";
4235 }
4236
4237
4238 sub qemu_devicedelverify {
4239     my ($vmid, $deviceid) = @_;
4240
4241     # need to verify that the device is correctly removed as device_del
4242     # is async and empty return is not reliable
4243
4244     for (my $i = 0; $i <= 5; $i++) {
4245          my $devices_list = vm_devices_list($vmid);
4246          return 1 if !defined($devices_list->{$deviceid});
4247          sleep 1;
4248     }
4249
4250     die "error on hot-unplugging device '$deviceid'\n";
4251 }
4252
4253 sub qemu_findorcreatescsihw {
4254     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4255
4256     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4257
4258     my $scsihwid="$controller_prefix$controller";
4259     my $devices_list = vm_devices_list($vmid);
4260
4261     if(!defined($devices_list->{$scsihwid})) {
4262         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4263     }
4264
4265     return 1;
4266 }
4267
4268 sub qemu_deletescsihw {
4269     my ($conf, $vmid, $opt) = @_;
4270
4271     my $device = parse_drive($opt, $conf->{$opt});
4272
4273     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4274         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4275         return 1;
4276     }
4277
4278     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4279
4280     my $devices_list = vm_devices_list($vmid);
4281     foreach my $opt (keys %{$devices_list}) {
4282         if (PVE::QemuServer::is_valid_drivename($opt)) {
4283             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4284             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4285                 return 1;
4286             }
4287         }
4288     }
4289
4290     my $scsihwid="scsihw$controller";
4291
4292     vm_deviceunplug($vmid, $conf, $scsihwid);
4293
4294     return 1;
4295 }
4296
4297 sub qemu_add_pci_bridge {
4298     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4299
4300     my $bridges = {};
4301
4302     my $bridgeid;
4303
4304     print_pci_addr($device, $bridges, $arch, $machine_type);
4305
4306     while (my ($k, $v) = each %$bridges) {
4307         $bridgeid = $k;
4308     }
4309     return 1 if !defined($bridgeid) || $bridgeid < 1;
4310
4311     my $bridge = "pci.$bridgeid";
4312     my $devices_list = vm_devices_list($vmid);
4313
4314     if (!defined($devices_list->{$bridge})) {
4315         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4316     }
4317
4318     return 1;
4319 }
4320
4321 sub qemu_set_link_status {
4322     my ($vmid, $device, $up) = @_;
4323
4324     vm_mon_cmd($vmid, "set_link", name => $device,
4325                up => $up ? JSON::true : JSON::false);
4326 }
4327
4328 sub qemu_netdevadd {
4329     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4330
4331     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4332     my %options =  split(/[=,]/, $netdev);
4333
4334     vm_mon_cmd($vmid, "netdev_add",  %options);
4335     return 1;
4336 }
4337
4338 sub qemu_netdevdel {
4339     my ($vmid, $deviceid) = @_;
4340
4341     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4342 }
4343
4344 sub qemu_usb_hotplug {
4345     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4346
4347     return if !$device;
4348
4349     # remove the old one first
4350     vm_deviceunplug($vmid, $conf, $deviceid);
4351
4352     # check if xhci controller is necessary and available
4353     if ($device->{usb3}) {
4354
4355         my $devicelist = vm_devices_list($vmid);
4356
4357         if (!$devicelist->{xhci}) {
4358             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4359             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4360         }
4361     }
4362     my $d = parse_usb_device($device->{host});
4363     $d->{usb3} = $device->{usb3};
4364
4365     # add the new one
4366     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4367 }
4368
4369 sub qemu_cpu_hotplug {
4370     my ($vmid, $conf, $vcpus) = @_;
4371
4372     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4373
4374     my $sockets = 1;
4375     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4376     $sockets = $conf->{sockets} if  $conf->{sockets};
4377     my $cores = $conf->{cores} || 1;
4378     my $maxcpus = $sockets * $cores;
4379
4380     $vcpus = $maxcpus if !$vcpus;
4381
4382     die "you can't add more vcpus than maxcpus\n"
4383         if $vcpus > $maxcpus;
4384
4385     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4386
4387     if ($vcpus < $currentvcpus) {
4388
4389         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4390
4391             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4392                 qemu_devicedel($vmid, "cpu$i");
4393                 my $retry = 0;
4394                 my $currentrunningvcpus = undef;
4395                 while (1) {
4396                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4397                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4398                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4399                     $retry++;
4400                     sleep 1;
4401                 }
4402                 #update conf after each succesfull cpu unplug
4403                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4404                 PVE::QemuConfig->write_config($vmid, $conf);
4405             }
4406         } else {
4407             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4408         }
4409
4410         return;
4411     }
4412
4413     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4414     die "vcpus in running vm does not match its configuration\n"
4415         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4416
4417     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4418
4419         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4420             my $cpustr = print_cpu_device($conf, $i);
4421             qemu_deviceadd($vmid, $cpustr);
4422
4423             my $retry = 0;
4424             my $currentrunningvcpus = undef;
4425             while (1) {
4426                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4427                 last if scalar(@{$currentrunningvcpus}) == $i;
4428                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4429                 sleep 1;
4430                 $retry++;
4431             }
4432             #update conf after each succesfull cpu hotplug
4433             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4434             PVE::QemuConfig->write_config($vmid, $conf);
4435         }
4436     } else {
4437
4438         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4439             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4440         }
4441     }
4442 }
4443
4444 sub qemu_block_set_io_throttle {
4445     my ($vmid, $deviceid,
4446         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4447         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4448         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4449         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4450
4451     return if !check_running($vmid) ;
4452
4453     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4454         bps => int($bps),
4455         bps_rd => int($bps_rd),
4456         bps_wr => int($bps_wr),
4457         iops => int($iops),
4458         iops_rd => int($iops_rd),
4459         iops_wr => int($iops_wr),
4460         bps_max => int($bps_max),
4461         bps_rd_max => int($bps_rd_max),
4462         bps_wr_max => int($bps_wr_max),
4463         iops_max => int($iops_max),
4464         iops_rd_max => int($iops_rd_max),
4465         iops_wr_max => int($iops_wr_max),
4466         bps_max_length => int($bps_max_length),
4467         bps_rd_max_length => int($bps_rd_max_length),
4468         bps_wr_max_length => int($bps_wr_max_length),
4469         iops_max_length => int($iops_max_length),
4470         iops_rd_max_length => int($iops_rd_max_length),
4471         iops_wr_max_length => int($iops_wr_max_length),
4472     );
4473
4474 }
4475
4476 # old code, only used to shutdown old VM after update
4477 sub __read_avail {
4478     my ($fh, $timeout) = @_;
4479
4480     my $sel = new IO::Select;
4481     $sel->add($fh);
4482
4483     my $res = '';
4484     my $buf;
4485
4486     my @ready;
4487     while (scalar (@ready = $sel->can_read($timeout))) {
4488         my $count;
4489         if ($count = $fh->sysread($buf, 8192)) {
4490             if ($buf =~ /^(.*)\(qemu\) $/s) {
4491                 $res .= $1;
4492                 last;
4493             } else {
4494                 $res .= $buf;
4495             }
4496         } else {
4497             if (!defined($count)) {
4498                 die "$!\n";
4499             }
4500             last;
4501         }
4502     }
4503
4504     die "monitor read timeout\n" if !scalar(@ready);
4505
4506     return $res;
4507 }
4508
4509 sub qemu_block_resize {
4510     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4511
4512     my $running = check_running($vmid);
4513
4514     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4515
4516     return if !$running;
4517
4518     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4519
4520 }
4521
4522 sub qemu_volume_snapshot {
4523     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4524
4525     my $running = check_running($vmid);
4526
4527     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4528         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4529     } else {
4530         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4531     }
4532 }
4533
4534 sub qemu_volume_snapshot_delete {
4535     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4536
4537     my $running = check_running($vmid);
4538
4539     if($running) {
4540
4541         $running = undef;
4542         my $conf = PVE::QemuConfig->load_config($vmid);
4543         foreach_drive($conf, sub {
4544             my ($ds, $drive) = @_;
4545             $running = 1 if $drive->{file} eq $volid;
4546         });
4547     }
4548
4549     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4550         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4551     } else {
4552         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4553     }
4554 }
4555
4556 sub set_migration_caps {
4557     my ($vmid) = @_;
4558
4559     my $cap_ref = [];
4560
4561     my $enabled_cap = {
4562         "auto-converge" => 1,
4563         "xbzrle" => 1,
4564         "x-rdma-pin-all" => 0,
4565         "zero-blocks" => 0,
4566         "compress" => 0
4567     };
4568
4569     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4570
4571     for my $supported_capability (@$supported_capabilities) {
4572         push @$cap_ref, {
4573             capability => $supported_capability->{capability},
4574             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4575         };
4576     }
4577
4578     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4579 }
4580
4581 my $fast_plug_option = {
4582     'lock' => 1,
4583     'name' => 1,
4584     'onboot' => 1,
4585     'shares' => 1,
4586     'startup' => 1,
4587     'description' => 1,
4588     'protection' => 1,
4589     'vmstatestorage' => 1,
4590 };
4591
4592 # hotplug changes in [PENDING]
4593 # $selection hash can be used to only apply specified options, for
4594 # example: { cores => 1 } (only apply changed 'cores')
4595 # $errors ref is used to return error messages
4596 sub vmconfig_hotplug_pending {
4597     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4598
4599     my $defaults = load_defaults();
4600     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4601
4602     # commit values which do not have any impact on running VM first
4603     # Note: those option cannot raise errors, we we do not care about
4604     # $selection and always apply them.
4605
4606     my $add_error = sub {
4607         my ($opt, $msg) = @_;
4608         $errors->{$opt} = "hotplug problem - $msg";
4609     };
4610
4611     my $changes = 0;
4612     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4613         if ($fast_plug_option->{$opt}) {
4614             $conf->{$opt} = $conf->{pending}->{$opt};
4615             delete $conf->{pending}->{$opt};
4616             $changes = 1;
4617         }
4618     }
4619
4620     if ($changes) {
4621         PVE::QemuConfig->write_config($vmid, $conf);
4622         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4623     }
4624
4625     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4626
4627     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4628     while (my ($opt, $force) = each %$pending_delete_hash) {
4629         next if $selection && !$selection->{$opt};
4630         eval {
4631             if ($opt eq 'hotplug') {
4632                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4633             } elsif ($opt eq 'tablet') {
4634                 die "skip\n" if !$hotplug_features->{usb};
4635                 if ($defaults->{tablet}) {
4636                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4637                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4638                         if $arch eq 'aarch64';
4639                 } else {
4640                     vm_deviceunplug($vmid, $conf, 'tablet');
4641                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4642                 }
4643             } elsif ($opt =~ m/^usb\d+/) {
4644                 die "skip\n";
4645                 # since we cannot reliably hot unplug usb devices
4646                 # we are disabling it
4647                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4648                 vm_deviceunplug($vmid, $conf, $opt);
4649             } elsif ($opt eq 'vcpus') {
4650                 die "skip\n" if !$hotplug_features->{cpu};
4651                 qemu_cpu_hotplug($vmid, $conf, undef);
4652             } elsif ($opt eq 'balloon') {
4653                 # enable balloon device is not hotpluggable
4654                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4655                 # here we reset the ballooning value to memory
4656                 my $balloon = $conf->{memory} || $defaults->{memory};
4657                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4658             } elsif ($fast_plug_option->{$opt}) {
4659                 # do nothing
4660             } elsif ($opt =~ m/^net(\d+)$/) {
4661                 die "skip\n" if !$hotplug_features->{network};
4662                 vm_deviceunplug($vmid, $conf, $opt);
4663             } elsif (is_valid_drivename($opt)) {
4664                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4665                 vm_deviceunplug($vmid, $conf, $opt);
4666                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4667             } elsif ($opt =~ m/^memory$/) {
4668                 die "skip\n" if !$hotplug_features->{memory};
4669                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4670             } elsif ($opt eq 'cpuunits') {
4671                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4672             } elsif ($opt eq 'cpulimit') {
4673                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4674             } else {
4675                 die "skip\n";
4676             }
4677         };
4678         if (my $err = $@) {
4679             &$add_error($opt, $err) if $err ne "skip\n";
4680         } else {
4681             # save new config if hotplug was successful
4682             delete $conf->{$opt};
4683             vmconfig_undelete_pending_option($conf, $opt);
4684             PVE::QemuConfig->write_config($vmid, $conf);
4685             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4686         }
4687     }
4688
4689     my $apply_pending_cloudinit;
4690     $apply_pending_cloudinit = sub {
4691         my ($key, $value) = @_;
4692         $apply_pending_cloudinit = sub {}; # once is enough
4693
4694         my @cloudinit_opts = keys %$confdesc_cloudinit;
4695         foreach my $opt (keys %{$conf->{pending}}) {
4696             next if !grep { $_ eq $opt } @cloudinit_opts;
4697             $conf->{$opt} = delete $conf->{pending}->{$opt};
4698         }
4699
4700         my $new_conf = { %$conf };
4701         $new_conf->{$key} = $value;
4702         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4703     };
4704
4705     foreach my $opt (keys %{$conf->{pending}}) {
4706         next if $selection && !$selection->{$opt};
4707         my $value = $conf->{pending}->{$opt};
4708         eval {
4709             if ($opt eq 'hotplug') {
4710                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4711             } elsif ($opt eq 'tablet') {
4712                 die "skip\n" if !$hotplug_features->{usb};
4713                 if ($value == 1) {
4714                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4715                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4716                         if $arch eq 'aarch64';
4717                 } elsif ($value == 0) {
4718                     vm_deviceunplug($vmid, $conf, 'tablet');
4719                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4720                 }
4721             } elsif ($opt =~ m/^usb\d+$/) {
4722                 die "skip\n";
4723                 # since we cannot reliably hot unplug usb devices
4724                 # we are disabling it
4725                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4726                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4727                 die "skip\n" if !$d;
4728                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4729             } elsif ($opt eq 'vcpus') {
4730                 die "skip\n" if !$hotplug_features->{cpu};
4731                 qemu_cpu_hotplug($vmid, $conf, $value);
4732             } elsif ($opt eq 'balloon') {
4733                 # enable/disable balloning device is not hotpluggable
4734                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4735                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4736                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4737
4738                 # allow manual ballooning if shares is set to zero
4739                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4740                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4741                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4742                 }
4743             } elsif ($opt =~ m/^net(\d+)$/) {
4744                 # some changes can be done without hotplug
4745                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4746                                     $vmid, $opt, $value, $arch, $machine_type);
4747             } elsif (is_valid_drivename($opt)) {
4748                 # some changes can be done without hotplug
4749                 my $drive = parse_drive($opt, $value);
4750                 if (drive_is_cloudinit($drive)) {
4751                     &$apply_pending_cloudinit($opt, $value);
4752                 }
4753                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4754                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4755             } elsif ($opt =~ m/^memory$/) { #dimms
4756                 die "skip\n" if !$hotplug_features->{memory};
4757                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4758             } elsif ($opt eq 'cpuunits') {
4759                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4760             } elsif ($opt eq 'cpulimit') {
4761                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4762                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4763             } else {
4764                 die "skip\n";  # skip non-hot-pluggable options
4765             }
4766         };
4767         if (my $err = $@) {
4768             &$add_error($opt, $err) if $err ne "skip\n";
4769         } else {
4770             # save new config if hotplug was successful
4771             $conf->{$opt} = $value;
4772             delete $conf->{pending}->{$opt};
4773             PVE::QemuConfig->write_config($vmid, $conf);
4774             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4775         }
4776     }
4777 }
4778
4779 sub try_deallocate_drive {
4780     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4781
4782     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4783         my $volid = $drive->{file};
4784         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4785             my $sid = PVE::Storage::parse_volume_id($volid);
4786             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4787
4788             # check if the disk is really unused
4789             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4790                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4791             PVE::Storage::vdisk_free($storecfg, $volid);
4792             return 1;
4793         } else {
4794             # If vm is not owner of this disk remove from config
4795             return 1;
4796         }
4797     }
4798
4799     return undef;
4800 }
4801
4802 sub vmconfig_delete_or_detach_drive {
4803     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4804
4805     my $drive = parse_drive($opt, $conf->{$opt});
4806
4807     my $rpcenv = PVE::RPCEnvironment::get();
4808     my $authuser = $rpcenv->get_user();
4809
4810     if ($force) {
4811         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4812         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4813     } else {
4814         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4815     }
4816 }
4817
4818 sub vmconfig_apply_pending {
4819     my ($vmid, $conf, $storecfg) = @_;
4820
4821     # cold plug
4822
4823     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4824     while (my ($opt, $force) = each %$pending_delete_hash) {
4825         die "internal error" if $opt =~ m/^unused/;
4826         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4827         if (!defined($conf->{$opt})) {
4828             vmconfig_undelete_pending_option($conf, $opt);
4829             PVE::QemuConfig->write_config($vmid, $conf);
4830         } elsif (is_valid_drivename($opt)) {
4831             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4832             vmconfig_undelete_pending_option($conf, $opt);
4833             delete $conf->{$opt};
4834             PVE::QemuConfig->write_config($vmid, $conf);
4835         } else {
4836             vmconfig_undelete_pending_option($conf, $opt);
4837             delete $conf->{$opt};
4838             PVE::QemuConfig->write_config($vmid, $conf);
4839         }
4840     }
4841
4842     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4843
4844     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4845         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4846
4847         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4848             # skip if nothing changed
4849         } elsif (is_valid_drivename($opt)) {
4850             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4851                 if defined($conf->{$opt});
4852             $conf->{$opt} = $conf->{pending}->{$opt};
4853         } else {
4854             $conf->{$opt} = $conf->{pending}->{$opt};
4855         }
4856
4857         delete $conf->{pending}->{$opt};
4858         PVE::QemuConfig->write_config($vmid, $conf);
4859     }
4860 }
4861
4862 my $safe_num_ne = sub {
4863     my ($a, $b) = @_;
4864
4865     return 0 if !defined($a) && !defined($b);
4866     return 1 if !defined($a);
4867     return 1 if !defined($b);
4868
4869     return $a != $b;
4870 };
4871
4872 my $safe_string_ne = sub {
4873     my ($a, $b) = @_;
4874
4875     return 0 if !defined($a) && !defined($b);
4876     return 1 if !defined($a);
4877     return 1 if !defined($b);
4878
4879     return $a ne $b;
4880 };
4881
4882 sub vmconfig_update_net {
4883     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4884
4885     my $newnet = parse_net($value);
4886
4887     if ($conf->{$opt}) {
4888         my $oldnet = parse_net($conf->{$opt});
4889
4890         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4891             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4892             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4893             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4894
4895             # for non online change, we try to hot-unplug
4896             die "skip\n" if !$hotplug;
4897             vm_deviceunplug($vmid, $conf, $opt);
4898         } else {
4899
4900             die "internal error" if $opt !~ m/net(\d+)/;
4901             my $iface = "tap${vmid}i$1";
4902
4903             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4904                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4905                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4906                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4907                 PVE::Network::tap_unplug($iface);
4908                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4909             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4910                 # Rate can be applied on its own but any change above needs to
4911                 # include the rate in tap_plug since OVS resets everything.
4912                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4913             }
4914
4915             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4916                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4917             }
4918
4919             return 1;
4920         }
4921     }
4922
4923     if ($hotplug) {
4924         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4925     } else {
4926         die "skip\n";
4927     }
4928 }
4929
4930 sub vmconfig_update_disk {
4931     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4932
4933     # fixme: do we need force?
4934
4935     my $drive = parse_drive($opt, $value);
4936
4937     if ($conf->{$opt}) {
4938
4939         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4940
4941             my $media = $drive->{media} || 'disk';
4942             my $oldmedia = $old_drive->{media} || 'disk';
4943             die "unable to change media type\n" if $media ne $oldmedia;
4944
4945             if (!drive_is_cdrom($old_drive)) {
4946
4947                 if ($drive->{file} ne $old_drive->{file}) {
4948
4949                     die "skip\n" if !$hotplug;
4950
4951                     # unplug and register as unused
4952                     vm_deviceunplug($vmid, $conf, $opt);
4953                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4954
4955                 } else {
4956                     # update existing disk
4957
4958                     # skip non hotpluggable value
4959                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4960                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4961                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4962                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4963                         die "skip\n";
4964                     }
4965
4966                     # apply throttle
4967                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4968                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4969                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4970                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4971                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4972                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4973                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4974                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4975                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4976                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4977                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4978                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4979                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4980                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4981                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4982                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4983                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4984                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4985
4986                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4987                                                    ($drive->{mbps} || 0)*1024*1024,
4988                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4989                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4990                                                    $drive->{iops} || 0,
4991                                                    $drive->{iops_rd} || 0,
4992                                                    $drive->{iops_wr} || 0,
4993                                                    ($drive->{mbps_max} || 0)*1024*1024,
4994                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4995                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4996                                                    $drive->{iops_max} || 0,
4997                                                    $drive->{iops_rd_max} || 0,
4998                                                    $drive->{iops_wr_max} || 0,
4999                                                    $drive->{bps_max_length} || 1,
5000                                                    $drive->{bps_rd_max_length} || 1,
5001                                                    $drive->{bps_wr_max_length} || 1,
5002                                                    $drive->{iops_max_length} || 1,
5003                                                    $drive->{iops_rd_max_length} || 1,
5004                                                    $drive->{iops_wr_max_length} || 1);
5005
5006                     }
5007
5008                     return 1;
5009                 }
5010
5011             } else { # cdrom
5012
5013                 if ($drive->{file} eq 'none') {
5014                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5015                     if (drive_is_cloudinit($old_drive)) {
5016                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5017                     }
5018                 } else {
5019                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5020                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5021                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5022                 }
5023
5024                 return 1;
5025             }
5026         }
5027     }
5028
5029     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5030     # hotplug new disks
5031     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5032     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5033 }
5034
5035 sub vm_start {
5036     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5037         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5038
5039     PVE::QemuConfig->lock_config($vmid, sub {
5040         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5041
5042         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5043
5044         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5045
5046         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5047
5048         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5049             vmconfig_apply_pending($vmid, $conf, $storecfg);
5050             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5051         }
5052
5053         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5054
5055         my $defaults = load_defaults();
5056
5057         # set environment variable useful inside network script
5058         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5059
5060         my $local_volumes = {};
5061
5062         if ($targetstorage) {
5063             foreach_drive($conf, sub {
5064                 my ($ds, $drive) = @_;
5065
5066                 return if drive_is_cdrom($drive);
5067
5068                 my $volid = $drive->{file};
5069
5070                 return if !$volid;
5071
5072                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5073
5074                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5075                 return if $scfg->{shared};
5076                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5077             });
5078
5079             my $format = undef;
5080
5081             foreach my $opt (sort keys %$local_volumes) {
5082
5083                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5084                 my $drive = parse_drive($opt, $conf->{$opt});
5085
5086                 #if remote storage is specified, use default format
5087                 if ($targetstorage && $targetstorage ne "1") {
5088                     $storeid = $targetstorage;
5089                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5090                     $format = $defFormat;
5091                 } else {
5092                     #else we use same format than original
5093                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5094                     $format = qemu_img_format($scfg, $volid);
5095                 }
5096
5097                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5098                 my $newdrive = $drive;
5099                 $newdrive->{format} = $format;
5100                 $newdrive->{file} = $newvolid;
5101                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5102                 $local_volumes->{$opt} = $drivestr;
5103                 #pass drive to conf for command line
5104                 $conf->{$opt} = $drivestr;
5105             }
5106         }
5107
5108         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5109
5110         my $migrate_port = 0;
5111         my $migrate_uri;
5112         if ($statefile) {
5113             if ($statefile eq 'tcp') {
5114                 my $localip = "localhost";
5115                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5116                 my $nodename = PVE::INotify::nodename();
5117
5118                 if (!defined($migration_type)) {
5119                     if (defined($datacenterconf->{migration}->{type})) {
5120                         $migration_type = $datacenterconf->{migration}->{type};
5121                     } else {
5122                         $migration_type = 'secure';
5123                     }
5124                 }
5125
5126                 if ($migration_type eq 'insecure') {
5127                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5128                     if ($migrate_network_addr) {
5129                         $localip = $migrate_network_addr;
5130                     } else {
5131                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5132                     }
5133
5134                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5135                 }
5136
5137                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5138                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5139                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5140                 push @$cmd, '-incoming', $migrate_uri;
5141                 push @$cmd, '-S';
5142
5143             } elsif ($statefile eq 'unix') {
5144                 # should be default for secure migrations as a ssh TCP forward
5145                 # tunnel is not deterministic reliable ready and fails regurarly
5146                 # to set up in time, so use UNIX socket forwards
5147                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5148                 unlink $socket_addr;
5149
5150                 $migrate_uri = "unix:$socket_addr";
5151
5152                 push @$cmd, '-incoming', $migrate_uri;
5153                 push @$cmd, '-S';
5154
5155             } else {
5156                 push @$cmd, '-loadstate', $statefile;
5157             }
5158         } elsif ($paused) {
5159             push @$cmd, '-S';
5160         }
5161
5162         # host pci devices
5163         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5164           my $d = parse_hostpci($conf->{"hostpci$i"});
5165           next if !$d;
5166           my $pcidevices = $d->{pciid};
5167           foreach my $pcidevice (@$pcidevices) {
5168                 my $pciid = $pcidevice->{id};
5169
5170                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5171                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5172                 die "no pci device info for device '$pciid'\n" if !$info;
5173
5174                 if ($d->{mdev}) {
5175                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5176                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5177                 } else {
5178                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5179                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5180                     die "can't reset pci device '$pciid'\n"
5181                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5182                 }
5183           }
5184         }
5185
5186         PVE::Storage::activate_volumes($storecfg, $vollist);
5187
5188         if (!check_running($vmid, 1)) {
5189             eval {
5190                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5191                     outfunc => sub {}, errfunc => sub {});
5192             };
5193         }
5194
5195         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5196                                                   : $defaults->{cpuunits};
5197
5198         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5199         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5200
5201         my %properties = (
5202             Slice => 'qemu.slice',
5203             KillMode => 'none',
5204             CPUShares => $cpuunits
5205         );
5206
5207         if (my $cpulimit = $conf->{cpulimit}) {
5208             $properties{CPUQuota} = int($cpulimit * 100);
5209         }
5210         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5211
5212         my $run_qemu = sub {
5213             PVE::Tools::run_fork sub {
5214                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5215                 run_command($cmd, %run_params);
5216             };
5217         };
5218
5219         if ($conf->{hugepages}) {
5220
5221             my $code = sub {
5222                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5223                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5224
5225                 PVE::QemuServer::Memory::hugepages_mount();
5226                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5227
5228                 eval { $run_qemu->() };
5229                 if (my $err = $@) {
5230                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5231                     die $err;
5232                 }
5233
5234                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5235             };
5236             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5237
5238         } else {
5239             eval { $run_qemu->() };
5240         }
5241
5242         if (my $err = $@) {
5243             # deactivate volumes if start fails
5244             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5245             die "start failed: $err";
5246         }
5247
5248         print "migration listens on $migrate_uri\n" if $migrate_uri;
5249
5250         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5251             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5252             warn $@ if $@;
5253         }
5254
5255         #start nbd server for storage migration
5256         if ($targetstorage) {
5257             my $nodename = PVE::INotify::nodename();
5258             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5259             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5260             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5261             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5262
5263             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5264
5265             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5266
5267             foreach my $opt (sort keys %$local_volumes) {
5268                 my $volid = $local_volumes->{$opt};
5269                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5270                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5271                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5272             }
5273         }
5274
5275         if ($migratedfrom) {
5276             eval {
5277                 set_migration_caps($vmid);
5278             };
5279             warn $@ if $@;
5280
5281             if ($spice_port) {
5282                 print "spice listens on port $spice_port\n";
5283                 if ($spice_ticket) {
5284                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5285                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5286                 }
5287             }
5288
5289         } else {
5290             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5291                 if !$statefile && $conf->{balloon};
5292
5293             foreach my $opt (keys %$conf) {
5294                 next if $opt !~  m/^net\d+$/;
5295                 my $nicconf = parse_net($conf->{$opt});
5296                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5297             }
5298         }
5299
5300         vm_mon_cmd_nocheck($vmid, 'qom-set',
5301                     path => "machine/peripheral/balloon0",
5302                     property => "guest-stats-polling-interval",
5303                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5304
5305     });
5306 }
5307
5308 sub vm_mon_cmd {
5309     my ($vmid, $execute, %params) = @_;
5310
5311     my $cmd = { execute => $execute, arguments => \%params };
5312     vm_qmp_command($vmid, $cmd);
5313 }
5314
5315 sub vm_mon_cmd_nocheck {
5316     my ($vmid, $execute, %params) = @_;
5317
5318     my $cmd = { execute => $execute, arguments => \%params };
5319     vm_qmp_command($vmid, $cmd, 1);
5320 }
5321
5322 sub vm_qmp_command {
5323     my ($vmid, $cmd, $nocheck) = @_;
5324
5325     my $res;
5326
5327     my $timeout;
5328     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5329         $timeout = $cmd->{arguments}->{timeout};
5330         delete $cmd->{arguments}->{timeout};
5331     }
5332
5333     eval {
5334         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5335         my $sname = qmp_socket($vmid);
5336         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5337             my $qmpclient = PVE::QMPClient->new();
5338
5339             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5340         } else {
5341             die "unable to open monitor socket\n";
5342         }
5343     };
5344     if (my $err = $@) {
5345         syslog("err", "VM $vmid qmp command failed - $err");
5346         die $err;
5347     }
5348
5349     return $res;
5350 }
5351
5352 sub vm_human_monitor_command {
5353     my ($vmid, $cmdline) = @_;
5354
5355     my $res;
5356
5357     my $cmd = {
5358         execute => 'human-monitor-command',
5359         arguments => { 'command-line' => $cmdline},
5360     };
5361
5362     return vm_qmp_command($vmid, $cmd);
5363 }
5364
5365 sub vm_commandline {
5366     my ($storecfg, $vmid) = @_;
5367
5368     my $conf = PVE::QemuConfig->load_config($vmid);
5369
5370     my $defaults = load_defaults();
5371
5372     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5373
5374     return PVE::Tools::cmd2string($cmd);
5375 }
5376
5377 sub vm_reset {
5378     my ($vmid, $skiplock) = @_;
5379
5380     PVE::QemuConfig->lock_config($vmid, sub {
5381
5382         my $conf = PVE::QemuConfig->load_config($vmid);
5383
5384         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5385
5386         vm_mon_cmd($vmid, "system_reset");
5387     });
5388 }
5389
5390 sub get_vm_volumes {
5391     my ($conf) = @_;
5392
5393     my $vollist = [];
5394     foreach_volid($conf, sub {
5395         my ($volid, $attr) = @_;
5396
5397         return if $volid =~ m|^/|;
5398
5399         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5400         return if !$sid;
5401
5402         push @$vollist, $volid;
5403     });
5404
5405     return $vollist;
5406 }
5407
5408 sub vm_stop_cleanup {
5409     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5410
5411     eval {
5412
5413         if (!$keepActive) {
5414             my $vollist = get_vm_volumes($conf);
5415             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5416         }
5417
5418         foreach my $ext (qw(mon qmp pid vnc qga)) {
5419             unlink "/var/run/qemu-server/${vmid}.$ext";
5420         }
5421
5422         foreach my $key (keys %$conf) {
5423             next if $key !~ m/^hostpci(\d+)$/;
5424             my $hostpciindex = $1;
5425             my $d = parse_hostpci($conf->{$key});
5426             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5427
5428             foreach my $pci (@{$d->{pciid}}) {
5429                 my $pciid = $pci->{id};
5430                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5431             }
5432         }
5433
5434         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5435     };
5436     warn $@ if $@; # avoid errors - just warn
5437 }
5438
5439 # Note: use $nockeck to skip tests if VM configuration file exists.
5440 # We need that when migration VMs to other nodes (files already moved)
5441 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5442 sub vm_stop {
5443     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5444
5445     $force = 1 if !defined($force) && !$shutdown;
5446
5447     if ($migratedfrom){
5448         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5449         kill 15, $pid if $pid;
5450         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5451         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5452         return;
5453     }
5454
5455     PVE::QemuConfig->lock_config($vmid, sub {
5456
5457         my $pid = check_running($vmid, $nocheck);
5458         return if !$pid;
5459
5460         my $conf;
5461         if (!$nocheck) {
5462             $conf = PVE::QemuConfig->load_config($vmid);
5463             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5464             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5465                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5466                 $timeout = $opts->{down} if $opts->{down};
5467             }
5468         }
5469
5470         $timeout = 60 if !defined($timeout);
5471
5472         eval {
5473             if ($shutdown) {
5474                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5475                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5476                 } else {
5477                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5478                 }
5479             } else {
5480                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5481             }
5482         };
5483         my $err = $@;
5484
5485         if (!$err) {
5486             my $count = 0;
5487             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5488                 $count++;
5489                 sleep 1;
5490             }
5491
5492             if ($count >= $timeout) {
5493                 if ($force) {
5494                     warn "VM still running - terminating now with SIGTERM\n";
5495                     kill 15, $pid;
5496                 } else {
5497                     die "VM quit/powerdown failed - got timeout\n";
5498                 }
5499             } else {
5500                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5501                 return;
5502             }
5503         } else {
5504             if ($force) {
5505                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5506                 kill 15, $pid;
5507             } else {
5508                 die "VM quit/powerdown failed\n";
5509             }
5510         }
5511
5512         # wait again
5513         $timeout = 10;
5514
5515         my $count = 0;
5516         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5517             $count++;
5518             sleep 1;
5519         }
5520
5521         if ($count >= $timeout) {
5522             warn "VM still running - terminating now with SIGKILL\n";
5523             kill 9, $pid;
5524             sleep 1;
5525         }
5526
5527         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5528    });
5529 }
5530
5531 sub vm_suspend {
5532     my ($vmid, $skiplock) = @_;
5533
5534     PVE::QemuConfig->lock_config($vmid, sub {
5535
5536         my $conf = PVE::QemuConfig->load_config($vmid);
5537
5538         PVE::QemuConfig->check_lock($conf)
5539             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5540
5541         vm_mon_cmd($vmid, "stop");
5542     });
5543 }
5544
5545 sub vm_resume {
5546     my ($vmid, $skiplock, $nocheck) = @_;
5547
5548     PVE::QemuConfig->lock_config($vmid, sub {
5549
5550         my $res = vm_mon_cmd($vmid, 'query-status');
5551         my $resume_cmd = 'cont';
5552
5553         if ($res->{status} && $res->{status} eq 'suspended') {
5554             $resume_cmd = 'system_wakeup';
5555         }
5556
5557         if (!$nocheck) {
5558
5559             my $conf = PVE::QemuConfig->load_config($vmid);
5560
5561             PVE::QemuConfig->check_lock($conf)
5562                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5563
5564             vm_mon_cmd($vmid, $resume_cmd);
5565
5566         } else {
5567             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5568         }
5569     });
5570 }
5571
5572 sub vm_sendkey {
5573     my ($vmid, $skiplock, $key) = @_;
5574
5575     PVE::QemuConfig->lock_config($vmid, sub {
5576
5577         my $conf = PVE::QemuConfig->load_config($vmid);
5578
5579         # there is no qmp command, so we use the human monitor command
5580         vm_human_monitor_command($vmid, "sendkey $key");
5581     });
5582 }
5583
5584 sub vm_destroy {
5585     my ($storecfg, $vmid, $skiplock) = @_;
5586
5587     PVE::QemuConfig->lock_config($vmid, sub {
5588
5589         my $conf = PVE::QemuConfig->load_config($vmid);
5590
5591         if (!check_running($vmid)) {
5592             destroy_vm($storecfg, $vmid, undef, $skiplock);
5593         } else {
5594             die "VM $vmid is running - destroy failed\n";
5595         }
5596     });
5597 }
5598
5599 # vzdump restore implementaion
5600
5601 sub tar_archive_read_firstfile {
5602     my $archive = shift;
5603
5604     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5605
5606     # try to detect archive type first
5607     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5608         die "unable to open file '$archive'\n";
5609     my $firstfile = <$fh>;
5610     kill 15, $pid;
5611     close $fh;
5612
5613     die "ERROR: archive contaions no data\n" if !$firstfile;
5614     chomp $firstfile;
5615
5616     return $firstfile;
5617 }
5618
5619 sub tar_restore_cleanup {
5620     my ($storecfg, $statfile) = @_;
5621
5622     print STDERR "starting cleanup\n";
5623
5624     if (my $fd = IO::File->new($statfile, "r")) {
5625         while (defined(my $line = <$fd>)) {
5626             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5627                 my $volid = $2;
5628                 eval {
5629                     if ($volid =~ m|^/|) {
5630                         unlink $volid || die 'unlink failed\n';
5631                     } else {
5632                         PVE::Storage::vdisk_free($storecfg, $volid);
5633                     }
5634                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5635                 };
5636                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5637             } else {
5638                 print STDERR "unable to parse line in statfile - $line";
5639             }
5640         }
5641         $fd->close();
5642     }
5643 }
5644
5645 sub restore_archive {
5646     my ($archive, $vmid, $user, $opts) = @_;
5647
5648     my $format = $opts->{format};
5649     my $comp;
5650
5651     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5652         $format = 'tar' if !$format;
5653         $comp = 'gzip';
5654     } elsif ($archive =~ m/\.tar$/) {
5655         $format = 'tar' if !$format;
5656     } elsif ($archive =~ m/.tar.lzo$/) {
5657         $format = 'tar' if !$format;
5658         $comp = 'lzop';
5659     } elsif ($archive =~ m/\.vma$/) {
5660         $format = 'vma' if !$format;
5661     } elsif ($archive =~ m/\.vma\.gz$/) {
5662         $format = 'vma' if !$format;
5663         $comp = 'gzip';
5664     } elsif ($archive =~ m/\.vma\.lzo$/) {
5665         $format = 'vma' if !$format;
5666         $comp = 'lzop';
5667     } else {
5668         $format = 'vma' if !$format; # default
5669     }
5670
5671     # try to detect archive format
5672     if ($format eq 'tar') {
5673         return restore_tar_archive($archive, $vmid, $user, $opts);
5674     } else {
5675         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5676     }
5677 }
5678
5679 sub restore_update_config_line {
5680     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5681
5682     return if $line =~ m/^\#qmdump\#/;
5683     return if $line =~ m/^\#vzdump\#/;
5684     return if $line =~ m/^lock:/;
5685     return if $line =~ m/^unused\d+:/;
5686     return if $line =~ m/^parent:/;
5687     return if $line =~ m/^template:/; # restored VM is never a template
5688
5689     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5690     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5691         # try to convert old 1.X settings
5692         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5693         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5694             my ($model, $macaddr) = split(/\=/, $devconfig);
5695             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5696             my $net = {
5697                 model => $model,
5698                 bridge => "vmbr$ind",
5699                 macaddr => $macaddr,
5700             };
5701             my $netstr = print_net($net);
5702
5703             print $outfd "net$cookie->{netcount}: $netstr\n";
5704             $cookie->{netcount}++;
5705         }
5706     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5707         my ($id, $netstr) = ($1, $2);
5708         my $net = parse_net($netstr);
5709         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5710         $netstr = print_net($net);
5711         print $outfd "$id: $netstr\n";
5712     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5713         my $virtdev = $1;
5714         my $value = $3;
5715         my $di = parse_drive($virtdev, $value);
5716         if (defined($di->{backup}) && !$di->{backup}) {
5717             print $outfd "#$line";
5718         } elsif ($map->{$virtdev}) {
5719             delete $di->{format}; # format can change on restore
5720             $di->{file} = $map->{$virtdev};
5721             $value = print_drive($vmid, $di);
5722             print $outfd "$virtdev: $value\n";
5723         } else {
5724             print $outfd $line;
5725         }
5726     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5727         my $vmgenid = $1;
5728         if ($vmgenid ne '0') {
5729             # always generate a new vmgenid if there was a valid one setup
5730             $vmgenid = generate_uuid();
5731         }
5732         print $outfd "vmgenid: $vmgenid\n";
5733     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5734         my ($uuid, $uuid_str);
5735         UUID::generate($uuid);
5736         UUID::unparse($uuid, $uuid_str);
5737         my $smbios1 = parse_smbios1($2);
5738         $smbios1->{uuid} = $uuid_str;
5739         print $outfd $1.print_smbios1($smbios1)."\n";
5740     } else {
5741         print $outfd $line;
5742     }
5743 }
5744
5745 sub scan_volids {
5746     my ($cfg, $vmid) = @_;
5747
5748     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5749
5750     my $volid_hash = {};
5751     foreach my $storeid (keys %$info) {
5752         foreach my $item (@{$info->{$storeid}}) {
5753             next if !($item->{volid} && $item->{size});
5754             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5755             $volid_hash->{$item->{volid}} = $item;
5756         }
5757     }
5758
5759     return $volid_hash;
5760 }
5761
5762 sub is_volume_in_use {
5763     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5764
5765     my $path = PVE::Storage::path($storecfg, $volid);
5766
5767     my $scan_config = sub {
5768         my ($cref, $snapname) = @_;
5769
5770         foreach my $key (keys %$cref) {
5771             my $value = $cref->{$key};
5772             if (is_valid_drivename($key)) {
5773                 next if $skip_drive && $key eq $skip_drive;
5774                 my $drive = parse_drive($key, $value);
5775                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5776                 return 1 if $volid eq $drive->{file};
5777                 if ($drive->{file} =~ m!^/!) {
5778                     return 1 if $drive->{file} eq $path;
5779                 } else {
5780                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5781                     next if !$storeid;
5782                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5783                     next if !$scfg;
5784                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5785                 }
5786             }
5787         }
5788
5789         return 0;
5790     };
5791
5792     return 1 if &$scan_config($conf);
5793
5794     undef $skip_drive;
5795
5796     foreach my $snapname (keys %{$conf->{snapshots}}) {
5797         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5798     }
5799
5800     return 0;
5801 }
5802
5803 sub update_disksize {
5804     my ($vmid, $conf, $volid_hash) = @_;
5805
5806     my $changes;
5807     my $prefix = "VM $vmid:";
5808
5809     # used and unused disks
5810     my $referenced = {};
5811
5812     # Note: it is allowed to define multiple storages with same path (alias), so
5813     # we need to check both 'volid' and real 'path' (two different volid can point
5814     # to the same path).
5815
5816     my $referencedpath = {};
5817
5818     # update size info
5819     foreach my $opt (keys %$conf) {
5820         if (is_valid_drivename($opt)) {
5821             my $drive = parse_drive($opt, $conf->{$opt});
5822             my $volid = $drive->{file};
5823             next if !$volid;
5824
5825             $referenced->{$volid} = 1;
5826             if ($volid_hash->{$volid} &&
5827                 (my $path = $volid_hash->{$volid}->{path})) {
5828                 $referencedpath->{$path} = 1;
5829             }
5830
5831             next if drive_is_cdrom($drive);
5832             next if !$volid_hash->{$volid};
5833
5834             $drive->{size} = $volid_hash->{$volid}->{size};
5835             my $new = print_drive($vmid, $drive);
5836             if ($new ne $conf->{$opt}) {
5837                 $changes = 1;
5838                 $conf->{$opt} = $new;
5839                 print "$prefix update disk '$opt' information.\n";
5840             }
5841         }
5842     }
5843
5844     # remove 'unusedX' entry if volume is used
5845     foreach my $opt (keys %$conf) {
5846         next if $opt !~ m/^unused\d+$/;
5847         my $volid = $conf->{$opt};
5848         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5849         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5850             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5851             $changes = 1;
5852             delete $conf->{$opt};
5853         }
5854
5855         $referenced->{$volid} = 1;
5856         $referencedpath->{$path} = 1 if $path;
5857     }
5858
5859     foreach my $volid (sort keys %$volid_hash) {
5860         next if $volid =~ m/vm-$vmid-state-/;
5861         next if $referenced->{$volid};
5862         my $path = $volid_hash->{$volid}->{path};
5863         next if !$path; # just to be sure
5864         next if $referencedpath->{$path};
5865         $changes = 1;
5866         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5867         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5868         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5869     }
5870
5871     return $changes;
5872 }
5873
5874 sub rescan {
5875     my ($vmid, $nolock, $dryrun) = @_;
5876
5877     my $cfg = PVE::Storage::config();
5878
5879     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5880     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5881     foreach my $stor (keys %{$cfg->{ids}}) {
5882         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5883     }
5884
5885     print "rescan volumes...\n";
5886     my $volid_hash = scan_volids($cfg, $vmid);
5887
5888     my $updatefn =  sub {
5889         my ($vmid) = @_;
5890
5891         my $conf = PVE::QemuConfig->load_config($vmid);
5892
5893         PVE::QemuConfig->check_lock($conf);
5894
5895         my $vm_volids = {};
5896         foreach my $volid (keys %$volid_hash) {
5897             my $info = $volid_hash->{$volid};
5898             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5899         }
5900
5901         my $changes = update_disksize($vmid, $conf, $vm_volids);
5902
5903         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5904     };
5905
5906     if (defined($vmid)) {
5907         if ($nolock) {
5908             &$updatefn($vmid);
5909         } else {
5910             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5911         }
5912     } else {
5913         my $vmlist = config_list();
5914         foreach my $vmid (keys %$vmlist) {
5915             if ($nolock) {
5916                 &$updatefn($vmid);
5917             } else {
5918                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5919             }
5920         }
5921     }
5922 }
5923
5924 sub restore_vma_archive {
5925     my ($archive, $vmid, $user, $opts, $comp) = @_;
5926
5927     my $readfrom = $archive;
5928
5929     my $cfg = PVE::Storage::config();
5930     my $commands = [];
5931     my $bwlimit = $opts->{bwlimit};
5932
5933     my $dbg_cmdstring = '';
5934     my $add_pipe = sub {
5935         my ($cmd) = @_;
5936         push @$commands, $cmd;
5937         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5938         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5939         $readfrom = '-';
5940     };
5941
5942     my $input = undef;
5943     if ($archive eq '-') {
5944         $input = '<&STDIN';
5945     } else {
5946         # If we use a backup from a PVE defined storage we also consider that
5947         # storage's rate limit:
5948         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5949         if (defined($volid)) {
5950             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5951             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5952             if ($readlimit) {
5953                 print STDERR "applying read rate limit: $readlimit\n";
5954                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5955                 $add_pipe->($cstream);
5956             }
5957         }
5958     }
5959
5960     if ($comp) {
5961         my $cmd;
5962         if ($comp eq 'gzip') {
5963             $cmd = ['zcat', $readfrom];
5964         } elsif ($comp eq 'lzop') {
5965             $cmd = ['lzop', '-d', '-c', $readfrom];
5966         } else {
5967             die "unknown compression method '$comp'\n";
5968         }
5969         $add_pipe->($cmd);
5970     }
5971
5972     my $tmpdir = "/var/tmp/vzdumptmp$$";
5973     rmtree $tmpdir;
5974
5975     # disable interrupts (always do cleanups)
5976     local $SIG{INT} =
5977         local $SIG{TERM} =
5978         local $SIG{QUIT} =
5979         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5980
5981     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5982     POSIX::mkfifo($mapfifo, 0600);
5983     my $fifofh;
5984
5985     my $openfifo = sub {
5986         open($fifofh, '>', $mapfifo) || die $!;
5987     };
5988
5989     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5990
5991     my $oldtimeout;
5992     my $timeout = 5;
5993
5994     my $devinfo = {};
5995
5996     my $rpcenv = PVE::RPCEnvironment::get();
5997
5998     my $conffile = PVE::QemuConfig->config_file($vmid);
5999     my $tmpfn = "$conffile.$$.tmp";
6000
6001     # Note: $oldconf is undef if VM does not exists
6002     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6003     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6004
6005     my %storage_limits;
6006
6007     my $print_devmap = sub {
6008         my $virtdev_hash = {};
6009
6010         my $cfgfn = "$tmpdir/qemu-server.conf";
6011
6012         # we can read the config - that is already extracted
6013         my $fh = IO::File->new($cfgfn, "r") ||
6014             "unable to read qemu-server.conf - $!\n";
6015
6016         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6017         if (-f $fwcfgfn) {
6018             my $pve_firewall_dir = '/etc/pve/firewall';
6019             mkdir $pve_firewall_dir; # make sure the dir exists
6020             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6021         }
6022
6023         while (defined(my $line = <$fh>)) {
6024             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6025                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6026                 die "archive does not contain data for drive '$virtdev'\n"
6027                     if !$devinfo->{$devname};
6028                 if (defined($opts->{storage})) {
6029                     $storeid = $opts->{storage} || 'local';
6030                 } elsif (!$storeid) {
6031                     $storeid = 'local';
6032                 }
6033                 $format = 'raw' if !$format;
6034                 $devinfo->{$devname}->{devname} = $devname;
6035                 $devinfo->{$devname}->{virtdev} = $virtdev;
6036                 $devinfo->{$devname}->{format} = $format;
6037                 $devinfo->{$devname}->{storeid} = $storeid;
6038
6039                 # check permission on storage
6040                 my $pool = $opts->{pool}; # todo: do we need that?
6041                 if ($user ne 'root@pam') {
6042                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6043                 }
6044
6045                 $storage_limits{$storeid} = $bwlimit;
6046
6047                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6048             }
6049         }
6050
6051         foreach my $key (keys %storage_limits) {
6052             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6053             next if !$limit;
6054             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6055             $storage_limits{$key} = $limit * 1024;
6056         }
6057
6058         foreach my $devname (keys %$devinfo) {
6059             die "found no device mapping information for device '$devname'\n"
6060                 if !$devinfo->{$devname}->{virtdev};
6061         }
6062
6063         # create empty/temp config
6064         if ($oldconf) {
6065             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6066             foreach_drive($oldconf, sub {
6067                 my ($ds, $drive) = @_;
6068
6069                 return if drive_is_cdrom($drive);
6070
6071                 my $volid = $drive->{file};
6072
6073                 return if !$volid || $volid =~ m|^/|;
6074
6075                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6076                 return if !$path || !$owner || ($owner != $vmid);
6077
6078                 # Note: only delete disk we want to restore
6079                 # other volumes will become unused
6080                 if ($virtdev_hash->{$ds}) {
6081                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6082                     if (my $err = $@) {
6083                         warn $err;
6084                     }
6085                 }
6086             });
6087
6088             # delete vmstate files
6089             # since after the restore we have no snapshots anymore
6090             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6091                 my $snap = $oldconf->{snapshots}->{$snapname};
6092                 if ($snap->{vmstate}) {
6093                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6094                     if (my $err = $@) {
6095                         warn $err;
6096                     }
6097                 }
6098             }
6099         }
6100
6101         my $map = {};
6102         foreach my $virtdev (sort keys %$virtdev_hash) {
6103             my $d = $virtdev_hash->{$virtdev};
6104             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6105             my $storeid = $d->{storeid};
6106             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6107
6108             my $map_opts = '';
6109             if (my $limit = $storage_limits{$storeid}) {
6110                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6111             }
6112
6113             # test if requested format is supported
6114             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6115             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6116             $d->{format} = $defFormat if !$supported;
6117
6118             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6119                                                   $d->{format}, undef, $alloc_size);
6120             print STDERR "new volume ID is '$volid'\n";
6121             $d->{volid} = $volid;
6122             my $path = PVE::Storage::path($cfg, $volid);
6123
6124             PVE::Storage::activate_volumes($cfg,[$volid]);
6125
6126             my $write_zeros = 1;
6127             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6128                 $write_zeros = 0;
6129             }
6130
6131             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6132
6133             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6134             $map->{$virtdev} = $volid;
6135         }
6136
6137         $fh->seek(0, 0) || die "seek failed - $!\n";
6138
6139         my $outfd = new IO::File ($tmpfn, "w") ||
6140             die "unable to write config for VM $vmid\n";
6141
6142         my $cookie = { netcount => 0 };
6143         while (defined(my $line = <$fh>)) {
6144             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6145         }
6146
6147         $fh->close();
6148         $outfd->close();
6149     };
6150
6151     eval {
6152         # enable interrupts
6153         local $SIG{INT} =
6154             local $SIG{TERM} =
6155             local $SIG{QUIT} =
6156             local $SIG{HUP} =
6157             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6158         local $SIG{ALRM} = sub { die "got timeout\n"; };
6159
6160         $oldtimeout = alarm($timeout);
6161
6162         my $parser = sub {
6163             my $line = shift;
6164
6165             print "$line\n";
6166
6167             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6168                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6169                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6170             } elsif ($line =~ m/^CTIME: /) {
6171                 # we correctly received the vma config, so we can disable
6172                 # the timeout now for disk allocation (set to 10 minutes, so
6173                 # that we always timeout if something goes wrong)
6174                 alarm(600);
6175                 &$print_devmap();
6176                 print $fifofh "done\n";
6177                 my $tmp = $oldtimeout || 0;
6178                 $oldtimeout = undef;
6179                 alarm($tmp);
6180                 close($fifofh);
6181             }
6182         };
6183
6184         print "restore vma archive: $dbg_cmdstring\n";
6185         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6186     };
6187     my $err = $@;
6188
6189     alarm($oldtimeout) if $oldtimeout;
6190
6191     my $vollist = [];
6192     foreach my $devname (keys %$devinfo) {
6193         my $volid = $devinfo->{$devname}->{volid};
6194         push @$vollist, $volid if $volid;
6195     }
6196
6197     PVE::Storage::deactivate_volumes($cfg, $vollist);
6198
6199     unlink $mapfifo;
6200
6201     if ($err) {
6202         rmtree $tmpdir;
6203         unlink $tmpfn;
6204
6205         foreach my $devname (keys %$devinfo) {
6206             my $volid = $devinfo->{$devname}->{volid};
6207             next if !$volid;
6208             eval {
6209                 if ($volid =~ m|^/|) {
6210                     unlink $volid || die 'unlink failed\n';
6211                 } else {
6212                     PVE::Storage::vdisk_free($cfg, $volid);
6213                 }
6214                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6215             };
6216             print STDERR "unable to cleanup '$volid' - $@" if $@;
6217         }
6218         die $err;
6219     }
6220
6221     rmtree $tmpdir;
6222
6223     rename($tmpfn, $conffile) ||
6224         die "unable to commit configuration file '$conffile'\n";
6225
6226     PVE::Cluster::cfs_update(); # make sure we read new file
6227
6228     eval { rescan($vmid, 1); };
6229     warn $@ if $@;
6230 }
6231
6232 sub restore_tar_archive {
6233     my ($archive, $vmid, $user, $opts) = @_;
6234
6235     if ($archive ne '-') {
6236         my $firstfile = tar_archive_read_firstfile($archive);
6237         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6238             if $firstfile ne 'qemu-server.conf';
6239     }
6240
6241     my $storecfg = PVE::Storage::config();
6242
6243     # destroy existing data - keep empty config
6244     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6245     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6246
6247     my $tocmd = "/usr/lib/qemu-server/qmextract";
6248
6249     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6250     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6251     $tocmd .= ' --prealloc' if $opts->{prealloc};
6252     $tocmd .= ' --info' if $opts->{info};
6253
6254     # tar option "xf" does not autodetect compression when read from STDIN,
6255     # so we pipe to zcat
6256     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6257         PVE::Tools::shellquote("--to-command=$tocmd");
6258
6259     my $tmpdir = "/var/tmp/vzdumptmp$$";
6260     mkpath $tmpdir;
6261
6262     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6263     local $ENV{VZDUMP_VMID} = $vmid;
6264     local $ENV{VZDUMP_USER} = $user;
6265
6266     my $conffile = PVE::QemuConfig->config_file($vmid);
6267     my $tmpfn = "$conffile.$$.tmp";
6268
6269     # disable interrupts (always do cleanups)
6270     local $SIG{INT} =
6271         local $SIG{TERM} =
6272         local $SIG{QUIT} =
6273         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6274
6275     eval {
6276         # enable interrupts
6277         local $SIG{INT} =
6278             local $SIG{TERM} =
6279             local $SIG{QUIT} =
6280             local $SIG{HUP} =
6281             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6282
6283         if ($archive eq '-') {
6284             print "extracting archive from STDIN\n";
6285             run_command($cmd, input => "<&STDIN");
6286         } else {
6287             print "extracting archive '$archive'\n";
6288             run_command($cmd);
6289         }
6290
6291         return if $opts->{info};
6292
6293         # read new mapping
6294         my $map = {};
6295         my $statfile = "$tmpdir/qmrestore.stat";
6296         if (my $fd = IO::File->new($statfile, "r")) {
6297             while (defined (my $line = <$fd>)) {
6298                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6299                     $map->{$1} = $2 if $1;
6300                 } else {
6301                     print STDERR "unable to parse line in statfile - $line\n";
6302                 }
6303             }
6304             $fd->close();
6305         }
6306
6307         my $confsrc = "$tmpdir/qemu-server.conf";
6308
6309         my $srcfd = new IO::File($confsrc, "r") ||
6310             die "unable to open file '$confsrc'\n";
6311
6312         my $outfd = new IO::File ($tmpfn, "w") ||
6313             die "unable to write config for VM $vmid\n";
6314
6315         my $cookie = { netcount => 0 };
6316         while (defined (my $line = <$srcfd>)) {
6317             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6318         }
6319
6320         $srcfd->close();
6321         $outfd->close();
6322     };
6323     my $err = $@;
6324
6325     if ($err) {
6326
6327         unlink $tmpfn;
6328
6329         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6330
6331         die $err;
6332     }
6333
6334     rmtree $tmpdir;
6335
6336     rename $tmpfn, $conffile ||
6337         die "unable to commit configuration file '$conffile'\n";
6338
6339     PVE::Cluster::cfs_update(); # make sure we read new file
6340
6341     eval { rescan($vmid, 1); };
6342     warn $@ if $@;
6343 };
6344
6345 sub foreach_storage_used_by_vm {
6346     my ($conf, $func) = @_;
6347
6348     my $sidhash = {};
6349
6350     foreach_drive($conf, sub {
6351         my ($ds, $drive) = @_;
6352         return if drive_is_cdrom($drive);
6353
6354         my $volid = $drive->{file};
6355
6356         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6357         $sidhash->{$sid} = $sid if $sid;
6358     });
6359
6360     foreach my $sid (sort keys %$sidhash) {
6361         &$func($sid);
6362     }
6363 }
6364
6365 sub do_snapshots_with_qemu {
6366     my ($storecfg, $volid) = @_;
6367
6368     my $storage_name = PVE::Storage::parse_volume_id($volid);
6369
6370     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6371         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6372         return 1;
6373     }
6374
6375     if ($volid =~ m/\.(qcow2|qed)$/){
6376         return 1;
6377     }
6378
6379     return undef;
6380 }
6381
6382 sub qga_check_running {
6383     my ($vmid, $nowarn) = @_;
6384
6385     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6386     if ($@) {
6387         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6388         return 0;
6389     }
6390     return 1;
6391 }
6392
6393 sub template_create {
6394     my ($vmid, $conf, $disk) = @_;
6395
6396     my $storecfg = PVE::Storage::config();
6397
6398     foreach_drive($conf, sub {
6399         my ($ds, $drive) = @_;
6400
6401         return if drive_is_cdrom($drive);
6402         return if $disk && $ds ne $disk;
6403
6404         my $volid = $drive->{file};
6405         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6406
6407         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6408         $drive->{file} = $voliddst;
6409         $conf->{$ds} = print_drive($vmid, $drive);
6410         PVE::QemuConfig->write_config($vmid, $conf);
6411     });
6412 }
6413
6414 sub qemu_img_convert {
6415     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6416
6417     my $storecfg = PVE::Storage::config();
6418     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6419     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6420
6421     if ($src_storeid && $dst_storeid) {
6422
6423         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6424
6425         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6426         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6427
6428         my $src_format = qemu_img_format($src_scfg, $src_volname);
6429         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6430
6431         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6432         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6433
6434         my $cmd = [];
6435         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6436         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6437         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6438         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6439         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6440         if ($is_zero_initialized) {
6441             push @$cmd, "zeroinit:$dst_path";
6442         } else {
6443             push @$cmd, $dst_path;
6444         }
6445
6446         my $parser = sub {
6447             my $line = shift;
6448             if($line =~ m/\((\S+)\/100\%\)/){
6449                 my $percent = $1;
6450                 my $transferred = int($size * $percent / 100);
6451                 my $remaining = $size - $transferred;
6452
6453                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6454             }
6455
6456         };
6457
6458         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6459         my $err = $@;
6460         die "copy failed: $err" if $err;
6461     }
6462 }
6463
6464 sub qemu_img_format {
6465     my ($scfg, $volname) = @_;
6466
6467     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6468         return $1;
6469     } else {
6470         return "raw";
6471     }
6472 }
6473
6474 sub qemu_drive_mirror {
6475     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6476
6477     $jobs = {} if !$jobs;
6478
6479     my $qemu_target;
6480     my $format;
6481     $jobs->{"drive-$drive"} = {};
6482
6483     if ($dst_volid =~ /^nbd:/) {
6484         $qemu_target = $dst_volid;
6485         $format = "nbd";
6486     } else {
6487         my $storecfg = PVE::Storage::config();
6488         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6489
6490         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6491
6492         $format = qemu_img_format($dst_scfg, $dst_volname);
6493
6494         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6495
6496         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6497     }
6498
6499     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6500     $opts->{format} = $format if $format;
6501
6502     print "drive mirror is starting for drive-$drive\n";
6503
6504     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6505
6506     if (my $err = $@) {
6507         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6508         die "mirroring error: $err";
6509     }
6510
6511     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6512 }
6513
6514 sub qemu_drive_mirror_monitor {
6515     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6516
6517     eval {
6518         my $err_complete = 0;
6519
6520         while (1) {
6521             die "storage migration timed out\n" if $err_complete > 300;
6522
6523             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6524
6525             my $running_mirror_jobs = {};
6526             foreach my $stat (@$stats) {
6527                 next if $stat->{type} ne 'mirror';
6528                 $running_mirror_jobs->{$stat->{device}} = $stat;
6529             }
6530
6531             my $readycounter = 0;
6532
6533             foreach my $job (keys %$jobs) {
6534
6535                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6536                     print "$job : finished\n";
6537                     delete $jobs->{$job};
6538                     next;
6539                 }
6540
6541                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6542
6543                 my $busy = $running_mirror_jobs->{$job}->{busy};
6544                 my $ready = $running_mirror_jobs->{$job}->{ready};
6545                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6546                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6547                     my $remaining = $total - $transferred;
6548                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6549
6550                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6551                 }
6552
6553                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6554             }
6555
6556             last if scalar(keys %$jobs) == 0;
6557
6558             if ($readycounter == scalar(keys %$jobs)) {
6559                 print "all mirroring jobs are ready \n";
6560                 last if $skipcomplete; #do the complete later
6561
6562                 if ($vmiddst && $vmiddst != $vmid) {
6563                     my $agent_running = $qga && qga_check_running($vmid);
6564                     if ($agent_running) {
6565                         print "freeze filesystem\n";
6566                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6567                     } else {
6568                         print "suspend vm\n";
6569                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6570                     }
6571
6572                     # if we clone a disk for a new target vm, we don't switch the disk
6573                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6574
6575                     if ($agent_running) {
6576                         print "unfreeze filesystem\n";
6577                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6578                     } else {
6579                         print "resume vm\n";
6580                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6581                     }
6582
6583                     last;
6584                 } else {
6585
6586                     foreach my $job (keys %$jobs) {
6587                         # try to switch the disk if source and destination are on the same guest
6588                         print "$job: Completing block job...\n";
6589
6590                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6591                         if ($@ =~ m/cannot be completed/) {
6592                             print "$job: Block job cannot be completed, try again.\n";
6593                             $err_complete++;
6594                         }else {
6595                             print "$job: Completed successfully.\n";
6596                             $jobs->{$job}->{complete} = 1;
6597                         }
6598                     }
6599                 }
6600             }
6601             sleep 1;
6602         }
6603     };
6604     my $err = $@;
6605
6606     if ($err) {
6607         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6608         die "mirroring error: $err";
6609     }
6610
6611 }
6612
6613 sub qemu_blockjobs_cancel {
6614     my ($vmid, $jobs) = @_;
6615
6616     foreach my $job (keys %$jobs) {
6617         print "$job: Cancelling block job\n";
6618         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6619         $jobs->{$job}->{cancel} = 1;
6620     }
6621
6622     while (1) {
6623         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6624
6625         my $running_jobs = {};
6626         foreach my $stat (@$stats) {
6627             $running_jobs->{$stat->{device}} = $stat;
6628         }
6629
6630         foreach my $job (keys %$jobs) {
6631
6632             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6633                 print "$job: Done.\n";
6634                 delete $jobs->{$job};
6635             }
6636         }
6637
6638         last if scalar(keys %$jobs) == 0;
6639
6640         sleep 1;
6641     }
6642 }
6643
6644 sub clone_disk {
6645     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6646         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6647
6648     my $newvolid;
6649
6650     if (!$full) {
6651         print "create linked clone of drive $drivename ($drive->{file})\n";
6652         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6653         push @$newvollist, $newvolid;
6654     } else {
6655
6656         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6657         $storeid = $storage if $storage;
6658
6659         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6660         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6661
6662         print "create full clone of drive $drivename ($drive->{file})\n";
6663         my $name = undef;
6664         if (drive_is_cloudinit($drive)) {
6665             $name = "vm-$newvmid-cloudinit";
6666             $snapname = undef;
6667             # cloudinit only supports raw and qcow2 atm:
6668             if ($dst_format eq 'qcow2') {
6669                 $name .= '.qcow2';
6670             } elsif ($dst_format ne 'raw') {
6671                 die "clone: unhandled format for cloudinit image\n";
6672             }
6673         }
6674         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6675         push @$newvollist, $newvolid;
6676
6677         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6678
6679         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6680         if (!$running || $snapname) {
6681             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6682         } else {
6683
6684             my $kvmver = get_running_qemu_version ($vmid);
6685             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6686                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6687                     if $drive->{iothread};
6688             }
6689
6690             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6691         }
6692     }
6693
6694     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6695
6696     my $disk = $drive;
6697     $disk->{format} = undef;
6698     $disk->{file} = $newvolid;
6699     $disk->{size} = $size;
6700
6701     return $disk;
6702 }
6703
6704 # this only works if VM is running
6705 sub get_current_qemu_machine {
6706     my ($vmid) = @_;
6707
6708     my $cmd = { execute => 'query-machines', arguments => {} };
6709     my $res = vm_qmp_command($vmid, $cmd);
6710
6711     my ($current, $default);
6712     foreach my $e (@$res) {
6713         $default = $e->{name} if $e->{'is-default'};
6714         $current = $e->{name} if $e->{'is-current'};
6715     }
6716
6717     # fallback to the default machine if current is not supported by qemu
6718     return $current || $default || 'pc';
6719 }
6720
6721 sub get_running_qemu_version {
6722     my ($vmid) = @_;
6723     my $cmd = { execute => 'query-version', arguments => {} };
6724     my $res = vm_qmp_command($vmid, $cmd);
6725     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6726 }
6727
6728 sub qemu_machine_feature_enabled {
6729     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6730
6731     my $current_major;
6732     my $current_minor;
6733
6734     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6735
6736         $current_major = $3;
6737         $current_minor = $4;
6738
6739     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6740
6741         $current_major = $1;
6742         $current_minor = $2;
6743     }
6744
6745     return 1 if $current_major > $version_major ||
6746                 ($current_major == $version_major &&
6747                  $current_minor >= $version_minor);
6748 }
6749
6750 sub qemu_machine_pxe {
6751     my ($vmid, $conf, $machine) = @_;
6752
6753     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6754
6755     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6756         $machine .= '.pxe';
6757     }
6758
6759     return $machine;
6760 }
6761
6762 sub qemu_use_old_bios_files {
6763     my ($machine_type) = @_;
6764
6765     return if !$machine_type;
6766
6767     my $use_old_bios_files = undef;
6768
6769     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6770         $machine_type = $1;
6771         $use_old_bios_files = 1;
6772     } else {
6773         my $kvmver = kvm_user_version();
6774         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6775         # load new efi bios files on migration. So this hack is required to allow
6776         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6777         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6778         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6779     }
6780
6781     return ($use_old_bios_files, $machine_type);
6782 }
6783
6784 sub create_efidisk($$$$$) {
6785     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6786
6787     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6788     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6789
6790     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6791     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6792     PVE::Storage::activate_volumes($storecfg, [$volid]);
6793
6794     my $path = PVE::Storage::path($storecfg, $volid);
6795     eval {
6796         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6797     };
6798     die "Copying EFI vars image failed: $@" if $@;
6799
6800     return ($volid, $vars_size);
6801 }
6802
6803 sub vm_iothreads_list {
6804     my ($vmid) = @_;
6805
6806     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6807
6808     my $iothreads = {};
6809     foreach my $iothread (@$res) {
6810         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6811     }
6812
6813     return $iothreads;
6814 }
6815
6816 sub scsihw_infos {
6817     my ($conf, $drive) = @_;
6818
6819     my $maxdev = 0;
6820
6821     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6822         $maxdev = 7;
6823     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6824         $maxdev = 1;
6825     } else {
6826         $maxdev = 256;
6827     }
6828
6829     my $controller = int($drive->{index} / $maxdev);
6830     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6831
6832     return ($maxdev, $controller, $controller_prefix);
6833 }
6834
6835 sub add_hyperv_enlightenments {
6836     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6837
6838     return if $winversion < 6;
6839     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6840
6841     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6842
6843     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6844         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6845         push @$cpuFlags , 'hv_vapic';
6846         push @$cpuFlags , 'hv_time';
6847     } else {
6848         push @$cpuFlags , 'hv_spinlocks=0xffff';
6849     }
6850
6851     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6852         push @$cpuFlags , 'hv_reset';
6853         push @$cpuFlags , 'hv_vpindex';
6854         push @$cpuFlags , 'hv_runtime';
6855     }
6856
6857     if ($winversion >= 7) {
6858         push @$cpuFlags , 'hv_relaxed';
6859
6860         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6861             push @$cpuFlags , 'hv_synic';
6862             push @$cpuFlags , 'hv_stimer';
6863         }
6864     }
6865 }
6866
6867 sub windows_version {
6868     my ($ostype) = @_;
6869
6870     return 0 if !$ostype;
6871
6872     my $winversion = 0;
6873
6874     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6875         $winversion = 5;
6876     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6877         $winversion = 6;
6878     } elsif ($ostype =~ m/^win(\d+)$/) {
6879         $winversion = $1;
6880     }
6881
6882     return $winversion;
6883 }
6884
6885 sub resolve_dst_disk_format {
6886         my ($storecfg, $storeid, $src_volname, $format) = @_;
6887         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6888
6889         if (!$format) {
6890             # if no target format is specified, use the source disk format as hint
6891             if ($src_volname) {
6892                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6893                 $format = qemu_img_format($scfg, $src_volname);
6894             } else {
6895                 return $defFormat;
6896             }
6897         }
6898
6899         # test if requested format is supported - else use default
6900         my $supported = grep { $_ eq $format } @$validFormats;
6901         $format = $defFormat if !$supported;
6902         return $format;
6903 }
6904
6905 sub resolve_first_disk {
6906     my $conf = shift;
6907     my @disks = PVE::QemuServer::valid_drive_names();
6908     my $firstdisk;
6909     foreach my $ds (reverse @disks) {
6910         next if !$conf->{$ds};
6911         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6912         next if PVE::QemuServer::drive_is_cdrom($disk);
6913         $firstdisk = $ds;
6914     }
6915     return $firstdisk;
6916 }
6917
6918 sub generate_uuid {
6919     my ($uuid, $uuid_str);
6920     UUID::generate($uuid);
6921     UUID::unparse($uuid, $uuid_str);
6922     return $uuid_str;
6923 }
6924
6925 sub generate_smbios1_uuid {
6926     return "uuid=".generate_uuid();
6927 }
6928
6929 sub nbd_stop {
6930     my ($vmid) = @_;
6931
6932     vm_mon_cmd($vmid, 'nbd-server-stop');
6933 }
6934
6935 # bash completion helper
6936
6937 sub complete_backup_archives {
6938     my ($cmdname, $pname, $cvalue) = @_;
6939
6940     my $cfg = PVE::Storage::config();
6941
6942     my $storeid;
6943
6944     if ($cvalue =~ m/^([^:]+):/) {
6945         $storeid = $1;
6946     }
6947
6948     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6949
6950     my $res = [];
6951     foreach my $id (keys %$data) {
6952         foreach my $item (@{$data->{$id}}) {
6953             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6954             push @$res, $item->{volid} if defined($item->{volid});
6955         }
6956     }
6957
6958     return $res;
6959 }
6960
6961 my $complete_vmid_full = sub {
6962     my ($running) = @_;
6963
6964     my $idlist = vmstatus();
6965
6966     my $res = [];
6967
6968     foreach my $id (keys %$idlist) {
6969         my $d = $idlist->{$id};
6970         if (defined($running)) {
6971             next if $d->{template};
6972             next if $running && $d->{status} ne 'running';
6973             next if !$running && $d->{status} eq 'running';
6974         }
6975         push @$res, $id;
6976
6977     }
6978     return $res;
6979 };
6980
6981 sub complete_vmid {
6982     return &$complete_vmid_full();
6983 }
6984
6985 sub complete_vmid_stopped {
6986     return &$complete_vmid_full(0);
6987 }
6988
6989 sub complete_vmid_running {
6990     return &$complete_vmid_full(1);
6991 }
6992
6993 sub complete_storage {
6994
6995     my $cfg = PVE::Storage::config();
6996     my $ids = $cfg->{ids};
6997
6998     my $res = [];
6999     foreach my $sid (keys %$ids) {
7000         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7001         next if !$ids->{$sid}->{content}->{images};
7002         push @$res, $sid;
7003     }
7004
7005     return $res;
7006 }
7007
7008 1;