PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use Time::HiRes qw(gettimeofday);
  40 use File::Copy qw(copy);
  41 use URI::Escape;
  42
  43 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  44 my $OVMF = {
  45     x86_64 => [
  46         "$EDK2_FW_BASE/OVMF_CODE.fd",
  47         "$EDK2_FW_BASE/OVMF_VARS.fd"
  48     ],
  49     aarch64 => [
  50         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  51         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  52     ],
  53 };
  54
  55 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  56
  57 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  58
  59 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  60
  61 # Note about locking: we use flock on the config file protect
  62 # against concurent actions.
  63 # Aditionaly, we have a 'lock' setting in the config file. This
  64 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  65 # allowed when such lock is set. But you can ignore this kind of
  66 # lock with the --skiplock flag.
  67
  68 cfs_register_file('/qemu-server/',
  69                   \&parse_vm_config,
  70                   \&write_vm_config);
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  73     description => "Some command save/restore state from this location.",
  74     type => 'string',
  75     maxLength => 128,
  76     optional => 1,
  77 });
  78
  79 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  80     description => "The name of the snapshot.",
  81     type => 'string', format => 'pve-configid',
  82     maxLength => 40,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $cpu_vendor_list = {
 123     # Intel CPUs
 124     486 => 'GenuineIntel',
 125     pentium => 'GenuineIntel',
 126     pentium2  => 'GenuineIntel',
 127     pentium3  => 'GenuineIntel',
 128     coreduo => 'GenuineIntel',
 129     core2duo => 'GenuineIntel',
 130     Conroe  => 'GenuineIntel',
 131     Penryn  => 'GenuineIntel',
 132     Nehalem  => 'GenuineIntel',
 133     'Nehalem-IBRS'  => 'GenuineIntel',
 134     Westmere => 'GenuineIntel',
 135     'Westmere-IBRS' => 'GenuineIntel',
 136     SandyBridge => 'GenuineIntel',
 137     'SandyBridge-IBRS' => 'GenuineIntel',
 138     IvyBridge => 'GenuineIntel',
 139     'IvyBridge-IBRS' => 'GenuineIntel',
 140     Haswell => 'GenuineIntel',
 141     'Haswell-IBRS' => 'GenuineIntel',
 142     'Haswell-noTSX' => 'GenuineIntel',
 143     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 144     Broadwell => 'GenuineIntel',
 145     'Broadwell-IBRS' => 'GenuineIntel',
 146     'Broadwell-noTSX' => 'GenuineIntel',
 147     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 148     'Skylake-Client' => 'GenuineIntel',
 149     'Skylake-Client-IBRS' => 'GenuineIntel',
 150     'Skylake-Server' => 'GenuineIntel',
 151     'Skylake-Server-IBRS' => 'GenuineIntel',
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 174
 175 my $cpu_fmt = {
 176     cputype => {
 177         description => "Emulated CPU type.",
 178         type => 'string',
 179         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 180         default => 'kvm64',
 181         default_key => 1,
 182     },
 183     hidden => {
 184         description => "Do not identify as a KVM virtual machine.",
 185         type => 'boolean',
 186         optional => 1,
 187         default => 0
 188     },
 189     flags => {
 190         description => "List of additional CPU flags separated by ';'."
 191                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 192                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 193         format_description => '+FLAG[;-FLAG...]',
 194         type => 'string',
 195         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 196         optional => 1,
 197     },
 198 };
 199
 200 my $watchdog_fmt = {
 201     model => {
 202         default_key => 1,
 203         type => 'string',
 204         enum => [qw(i6300esb ib700)],
 205         description => "Watchdog type to emulate.",
 206         default => 'i6300esb',
 207         optional => 1,
 208     },
 209     action => {
 210         type => 'string',
 211         enum => [qw(reset shutdown poweroff pause debug none)],
 212         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 213         optional => 1,
 214     },
 215 };
 216 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 217
 218 my $agent_fmt = {
 219     enabled => {
 220         description => "Enable/disable Qemu GuestAgent.",
 221         type => 'boolean',
 222         default => 0,
 223         default_key => 1,
 224     },
 225     fstrim_cloned_disks => {
 226         description => "Run fstrim after cloning/moving a disk.",
 227         type => 'boolean',
 228         optional => 1,
 229         default => 0
 230     },
 231 };
 232
 233 my $vga_fmt = {
 234     type => {
 235         description => "Select the VGA type.",
 236         type => 'string',
 237         default => 'std',
 238         optional => 1,
 239         default_key => 1,
 240         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 241     },
 242     memory => {
 243         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 244         type => 'integer',
 245         optional => 1,
 246         minimum => 4,
 247         maximum => 512,
 248     },
 249 };
 250
 251 my $confdesc = {
 252     onboot => {
 253         optional => 1,
 254         type => 'boolean',
 255         description => "Specifies whether a VM will be started during system bootup.",
 256         default => 0,
 257     },
 258     autostart => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Automatic restart after crash (currently ignored).",
 262         default => 0,
 263     },
 264     hotplug => {
 265         optional => 1,
 266         type => 'string', format => 'pve-hotplug-features',
 267         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 268         default => 'network,disk,usb',
 269     },
 270     reboot => {
 271         optional => 1,
 272         type => 'boolean',
 273         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 274         default => 1,
 275     },
 276     lock => {
 277         optional => 1,
 278         type => 'string',
 279         description => "Lock/unlock the VM.",
 280         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 281     },
 282     cpulimit => {
 283         optional => 1,
 284         type => 'number',
 285         description => "Limit of CPU usage.",
 286         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 287         minimum => 0,
 288         maximum => 128,
 289         default => 0,
 290     },
 291     cpuunits => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "CPU weight for a VM.",
 295         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 296         minimum => 2,
 297         maximum => 262144,
 298         default => 1024,
 299     },
 300     memory => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 304         minimum => 16,
 305         default => 512,
 306     },
 307     balloon => {
 308         optional => 1,
 309         type => 'integer',
 310         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 311         minimum => 0,
 312     },
 313     shares => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 317         minimum => 0,
 318         maximum => 50000,
 319         default => 1000,
 320     },
 321     keyboard => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 325                        "It should not be necessary to set it.",
 326         enum => PVE::Tools::kvmkeymaplist(),
 327         default => undef,
 328     },
 329     name => {
 330         optional => 1,
 331         type => 'string', format => 'dns-name',
 332         description => "Set a name for the VM. Only used on the configuration web interface.",
 333     },
 334     scsihw => {
 335         optional => 1,
 336         type => 'string',
 337         description => "SCSI controller model",
 338         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 339         default => 'lsi',
 340     },
 341     description => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 345     },
 346     ostype => {
 347         optional => 1,
 348         type => 'string',
 349         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 350         description => "Specify guest operating system.",
 351         verbose_description => <<EODESC,
 352 Specify guest operating system. This is used to enable special
 353 optimization/features for specific operating systems:
 354
 355 [horizontal]
 356 other;; unspecified OS
 357 wxp;; Microsoft Windows XP
 358 w2k;; Microsoft Windows 2000
 359 w2k3;; Microsoft Windows 2003
 360 w2k8;; Microsoft Windows 2008
 361 wvista;; Microsoft Windows Vista
 362 win7;; Microsoft Windows 7
 363 win8;; Microsoft Windows 8/2012/2012r2
 364 win10;; Microsoft Windows 10/2016
 365 l24;; Linux 2.4 Kernel
 366 l26;; Linux 2.6/3.X Kernel
 367 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 368 EODESC
 369     },
 370     boot => {
 371         optional => 1,
 372         type => 'string',
 373         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 374         pattern => '[acdn]{1,4}',
 375         default => 'cdn',
 376     },
 377     bootdisk => {
 378         optional => 1,
 379         type => 'string', format => 'pve-qm-bootdisk',
 380         description => "Enable booting from specified disk.",
 381         pattern => '(ide|sata|scsi|virtio)\d+',
 382     },
 383     smp => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPUs. Please use option -sockets instead.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     sockets => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of CPU sockets.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     cores => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "The number of cores per socket.",
 401         minimum => 1,
 402         default => 1,
 403     },
 404     numa => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Enable/disable NUMA.",
 408         default => 0,
 409     },
 410     hugepages => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Enable/disable hugepages memory.",
 414         enum => [qw(any 2 1024)],
 415     },
 416     vcpus => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "Number of hotplugged vcpus.",
 420         minimum => 1,
 421         default => 0,
 422     },
 423     acpi => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable ACPI.",
 427         default => 1,
 428     },
 429     agent => {
 430         optional => 1,
 431         description => "Enable/disable Qemu GuestAgent and its properties.",
 432         type => 'string',
 433         format => $agent_fmt,
 434     },
 435     kvm => {
 436         optional => 1,
 437         type => 'boolean',
 438         description => "Enable/disable KVM hardware virtualization.",
 439         default => 1,
 440     },
 441     tdf => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable time drift fix.",
 445         default => 0,
 446     },
 447     localtime => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 451     },
 452     freeze => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 456     },
 457     vga => {
 458         optional => 1,
 459         type => 'string', format => $vga_fmt,
 460         description => "Configure the VGA hardware.",
 461         verbose_description => "Configure the VGA Hardware. If you want to use ".
 462             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 463             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 464             "is 'std' for all OS types besides some Windows versions (XP and " .
 465             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 466             "display server. For win* OS you can select how many independent " .
 467             "displays you want, Linux guests can add displays them self.\n".
 468             "You can also run without any graphic card, using a serial device as terminal.",
 469     },
 470     watchdog => {
 471         optional => 1,
 472         type => 'string', format => 'pve-qm-watchdog',
 473         description => "Create a virtual hardware watchdog device.",
 474         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 475             " (by a guest action), the watchdog must be periodically polled " .
 476             "by an agent inside the guest or else the watchdog will reset " .
 477             "the guest (or execute the respective action specified)",
 478     },
 479     startdate => {
 480         optional => 1,
 481         type => 'string',
 482         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 483         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 484         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 485         default => 'now',
 486     },
 487     startup =>  get_standard_option('pve-startup-order'),
 488     template => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable Template.",
 492         default => 0,
 493     },
 494     args => {
 495         optional => 1,
 496         type => 'string',
 497         description => "Arbitrary arguments passed to kvm.",
 498         verbose_description => <<EODESCR,
 499 Arbitrary arguments passed to kvm, for example:
 500
 501 args: -no-reboot -no-hpet
 502
 503 NOTE: this option is for experts only.
 504 EODESCR
 505     },
 506     tablet => {
 507         optional => 1,
 508         type => 'boolean',
 509         default => 1,
 510         description => "Enable/disable the USB tablet device.",
 511         verbose_description => "Enable/disable the USB tablet device. This device is " .
 512             "usually needed to allow absolute mouse positioning with VNC. " .
 513             "Else the mouse runs out of sync with normal VNC clients. " .
 514             "If you're running lots of console-only guests on one host, " .
 515             "you may consider disabling this to save some context switches. " .
 516             "This is turned off by default if you use spice (-vga=qxl).",
 517     },
 518     migrate_speed => {
 519         optional => 1,
 520         type => 'integer',
 521         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 522         minimum => 0,
 523         default => 0,
 524     },
 525     migrate_downtime => {
 526         optional => 1,
 527         type => 'number',
 528         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 529         minimum => 0,
 530         default => 0.1,
 531     },
 532     cdrom => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-ide',
 535         typetext => '<volume>',
 536         description => "This is an alias for option -ide2",
 537     },
 538     cpu => {
 539         optional => 1,
 540         description => "Emulated CPU type.",
 541         type => 'string',
 542         format => $cpu_fmt,
 543     },
 544     parent => get_standard_option('pve-snapshot-name', {
 545         optional => 1,
 546         description => "Parent snapshot name. This is used internally, and should not be modified.",
 547     }),
 548     snaptime => {
 549         optional => 1,
 550         description => "Timestamp for snapshots.",
 551         type => 'integer',
 552         minimum => 0,
 553     },
 554     vmstate => {
 555         optional => 1,
 556         type => 'string', format => 'pve-volume-id',
 557         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 558     },
 559     vmstatestorage => get_standard_option('pve-storage-id', {
 560         description => "Default storage for VM state volumes/files.",
 561         optional => 1,
 562     }),
 563     runningmachine => get_standard_option('pve-qemu-machine', {
 564         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 565     }),
 566     machine => get_standard_option('pve-qemu-machine'),
 567     arch => {
 568         description => "Virtual processor architecture. Defaults to the host.",
 569         optional => 1,
 570         type => 'string',
 571         enum => [qw(x86_64 aarch64)],
 572     },
 573     smbios1 => {
 574         description => "Specify SMBIOS type 1 fields.",
 575         type => 'string', format => 'pve-qm-smbios1',
 576         maxLength => 256,
 577         optional => 1,
 578     },
 579     protection => {
 580         optional => 1,
 581         type => 'boolean',
 582         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 583         default => 0,
 584     },
 585     bios => {
 586         optional => 1,
 587         type => 'string',
 588         enum => [ qw(seabios ovmf) ],
 589         description => "Select BIOS implementation.",
 590         default => 'seabios',
 591     },
 592     vmgenid => {
 593         type => 'string',
 594         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 595         format_description => 'UUID',
 596         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 597         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 598             " 128-bit integer value identifier to the guest OS. This allows to".
 599             " notify the guest operating system when the virtual machine is".
 600             " executed with a different configuration (e.g. snapshot execution".
 601             " or creation from a template). The guest operating system notices".
 602             " the change, and is then able to react as appropriate by marking".
 603             " its copies of distributed databases as dirty, re-initializing its".
 604             " random number generator, etc.\n".
 605             "Note that auto-creation only works when done throug API/CLI create".
 606             " or update methods, but not when manually editing the config file.",
 607         default => "1 (autogenerated)",
 608         optional => 1,
 609     },
 610 };
 611
 612 my $confdesc_cloudinit = {
 613     citype => {
 614         optional => 1,
 615         type => 'string',
 616         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 617         enum => ['configdrive2', 'nocloud'],
 618     },
 619     ciuser => {
 620         optional => 1,
 621         type => 'string',
 622         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 623     },
 624     cipassword => {
 625         optional => 1,
 626         type => 'string',
 627         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 628     },
 629     searchdomain => {
 630         optional => 1,
 631         type => 'string',
 632         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 633     },
 634     nameserver => {
 635         optional => 1,
 636         type => 'string', format => 'address-list',
 637         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 638     },
 639     sshkeys => {
 640         optional => 1,
 641         type => 'string',
 642         format => 'urlencoded',
 643         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 644     },
 645 };
 646
 647 # what about other qemu settings ?
 648 #cpu => 'string',
 649 #machine => 'string',
 650 #fda => 'file',
 651 #fdb => 'file',
 652 #mtdblock => 'file',
 653 #sd => 'file',
 654 #pflash => 'file',
 655 #snapshot => 'bool',
 656 #bootp => 'file',
 657 ##tftp => 'dir',
 658 ##smb => 'dir',
 659 #kernel => 'file',
 660 #append => 'string',
 661 #initrd => 'file',
 662 ##soundhw => 'string',
 663
 664 while (my ($k, $v) = each %$confdesc) {
 665     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 666 }
 667
 668 my $MAX_IDE_DISKS = 4;
 669 my $MAX_SCSI_DISKS = 14;
 670 my $MAX_VIRTIO_DISKS = 16;
 671 my $MAX_SATA_DISKS = 6;
 672 my $MAX_USB_DEVICES = 5;
 673 my $MAX_NETS = 32;
 674 my $MAX_UNUSED_DISKS = 256;
 675 my $MAX_HOSTPCI_DEVICES = 4;
 676 my $MAX_SERIAL_PORTS = 4;
 677 my $MAX_PARALLEL_PORTS = 3;
 678 my $MAX_NUMA = 8;
 679
 680 my $numa_fmt = {
 681     cpus => {
 682         type => "string",
 683         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 684         description => "CPUs accessing this NUMA node.",
 685         format_description => "id[-id];...",
 686     },
 687     memory => {
 688         type => "number",
 689         description => "Amount of memory this NUMA node provides.",
 690         optional => 1,
 691     },
 692     hostnodes => {
 693         type => "string",
 694         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 695         description => "Host NUMA nodes to use.",
 696         format_description => "id[-id];...",
 697         optional => 1,
 698     },
 699     policy => {
 700         type => 'string',
 701         enum => [qw(preferred bind interleave)],
 702         description => "NUMA allocation policy.",
 703         optional => 1,
 704     },
 705 };
 706 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 707 my $numadesc = {
 708     optional => 1,
 709     type => 'string', format => $numa_fmt,
 710     description => "NUMA topology.",
 711 };
 712 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 713
 714 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 715     $confdesc->{"numa$i"} = $numadesc;
 716 }
 717
 718 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 719                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 720                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 721 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 722
 723 my $net_fmt_bridge_descr = <<__EOD__;
 724 Bridge to attach the network device to. The Proxmox VE standard bridge
 725 is called 'vmbr0'.
 726
 727 If you do not specify a bridge, we create a kvm user (NATed) network
 728 device, which provides DHCP and DNS services. The following addresses
 729 are used:
 730
 731  10.0.2.2   Gateway
 732  10.0.2.3   DNS Server
 733  10.0.2.4   SMB Server
 734
 735 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 736 __EOD__
 737
 738 my $net_fmt = {
 739     macaddr => {
 740         type => 'string',
 741         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 742         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 743         format_description => "XX:XX:XX:XX:XX:XX",
 744         optional => 1,
 745     },
 746     model => {
 747         type => 'string',
 748         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 749         enum => $nic_model_list,
 750         default_key => 1,
 751     },
 752     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 753     bridge => {
 754         type => 'string',
 755         description => $net_fmt_bridge_descr,
 756         format_description => 'bridge',
 757         optional => 1,
 758     },
 759     queues => {
 760         type => 'integer',
 761         minimum => 0, maximum => 16,
 762         description => 'Number of packet queues to be used on the device.',
 763         optional => 1,
 764     },
 765     rate => {
 766         type => 'number',
 767         minimum => 0,
 768         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 769         optional => 1,
 770     },
 771     tag => {
 772         type => 'integer',
 773         minimum => 1, maximum => 4094,
 774         description => 'VLAN tag to apply to packets on this interface.',
 775         optional => 1,
 776     },
 777     trunks => {
 778         type => 'string',
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => 'VLAN trunks to pass through this interface.',
 781         format_description => 'vlanid[;vlanid...]',
 782         optional => 1,
 783     },
 784     firewall => {
 785         type => 'boolean',
 786         description => 'Whether this interface should be protected by the firewall.',
 787         optional => 1,
 788     },
 789     link_down => {
 790         type => 'boolean',
 791         description => 'Whether this interface should be disconnected (like pulling the plug).',
 792         optional => 1,
 793     },
 794 };
 795
 796 my $netdesc = {
 797     optional => 1,
 798     type => 'string', format => $net_fmt,
 799     description => "Specify network devices.",
 800 };
 801
 802 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 803
 804 my $ipconfig_fmt = {
 805     ip => {
 806         type => 'string',
 807         format => 'pve-ipv4-config',
 808         format_description => 'IPv4Format/CIDR',
 809         description => 'IPv4 address in CIDR format.',
 810         optional => 1,
 811         default => 'dhcp',
 812     },
 813     gw => {
 814         type => 'string',
 815         format => 'ipv4',
 816         format_description => 'GatewayIPv4',
 817         description => 'Default gateway for IPv4 traffic.',
 818         optional => 1,
 819         requires => 'ip',
 820     },
 821     ip6 => {
 822         type => 'string',
 823         format => 'pve-ipv6-config',
 824         format_description => 'IPv6Format/CIDR',
 825         description => 'IPv6 address in CIDR format.',
 826         optional => 1,
 827         default => 'dhcp',
 828     },
 829     gw6 => {
 830         type => 'string',
 831         format => 'ipv6',
 832         format_description => 'GatewayIPv6',
 833         description => 'Default gateway for IPv6 traffic.',
 834         optional => 1,
 835         requires => 'ip6',
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 839 my $ipconfigdesc = {
 840     optional => 1,
 841     type => 'string', format => 'pve-qm-ipconfig',
 842     description => <<'EODESCR',
 843 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 844
 845 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 846
 847 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 848 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 849
 850 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 851 EODESCR
 852 };
 853 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 854
 855 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 856     $confdesc->{"net$i"} = $netdesc;
 857     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 858 }
 859
 860 foreach my $key (keys %$confdesc_cloudinit) {
 861     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 862 }
 863
 864 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 865 sub verify_volume_id_or_qm_path {
 866     my ($volid, $noerr) = @_;
 867
 868     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 869         return $volid;
 870     }
 871
 872     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 873     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 874     if ($@) {
 875         return undef if $noerr;
 876         die $@;
 877     }
 878     return $volid;
 879 }
 880
 881 my $drivename_hash;
 882
 883 my %drivedesc_base = (
 884     volume => { alias => 'file' },
 885     file => {
 886         type => 'string',
 887         format => 'pve-volume-id-or-qm-path',
 888         default_key => 1,
 889         format_description => 'volume',
 890         description => "The drive's backing volume.",
 891     },
 892     media => {
 893         type => 'string',
 894         enum => [qw(cdrom disk)],
 895         description => "The drive's media type.",
 896         default => 'disk',
 897         optional => 1
 898     },
 899     cyls => {
 900         type => 'integer',
 901         description => "Force the drive's physical geometry to have a specific cylinder count.",
 902         optional => 1
 903     },
 904     heads => {
 905         type => 'integer',
 906         description => "Force the drive's physical geometry to have a specific head count.",
 907         optional => 1
 908     },
 909     secs => {
 910         type => 'integer',
 911         description => "Force the drive's physical geometry to have a specific sector count.",
 912         optional => 1
 913     },
 914     trans => {
 915         type => 'string',
 916         enum => [qw(none lba auto)],
 917         description => "Force disk geometry bios translation mode.",
 918         optional => 1,
 919     },
 920     snapshot => {
 921         type => 'boolean',
 922         description => "Controls qemu's snapshot mode feature."
 923             . " If activated, changes made to the disk are temporary and will"
 924             . " be discarded when the VM is shutdown.",
 925         optional => 1,
 926     },
 927     cache => {
 928         type => 'string',
 929         enum => [qw(none writethrough writeback unsafe directsync)],
 930         description => "The drive's cache mode",
 931         optional => 1,
 932     },
 933     format => get_standard_option('pve-qm-image-format'),
 934     size => {
 935         type => 'string',
 936         format => 'disk-size',
 937         format_description => 'DiskSize',
 938         description => "Disk size. This is purely informational and has no effect.",
 939         optional => 1,
 940     },
 941     backup => {
 942         type => 'boolean',
 943         description => "Whether the drive should be included when making backups.",
 944         optional => 1,
 945     },
 946     replicate => {
 947         type => 'boolean',
 948         description => 'Whether the drive should considered for replication jobs.',
 949         optional => 1,
 950         default => 1,
 951     },
 952     rerror => {
 953         type => 'string',
 954         enum => [qw(ignore report stop)],
 955         description => 'Read error action.',
 956         optional => 1,
 957     },
 958     werror => {
 959         type => 'string',
 960         enum => [qw(enospc ignore report stop)],
 961         description => 'Write error action.',
 962         optional => 1,
 963     },
 964     aio => {
 965         type => 'string',
 966         enum => [qw(native threads)],
 967         description => 'AIO type to use.',
 968         optional => 1,
 969     },
 970     discard => {
 971         type => 'string',
 972         enum => [qw(ignore on)],
 973         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 974         optional => 1,
 975     },
 976     detect_zeroes => {
 977         type => 'boolean',
 978         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 979         optional => 1,
 980     },
 981     serial => {
 982         type => 'string',
 983         format => 'urlencoded',
 984         format_description => 'serial',
 985         maxLength => 20*3, # *3 since it's %xx url enoded
 986         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 987         optional => 1,
 988     },
 989     shared => {
 990         type => 'boolean',
 991         description => 'Mark this locally-managed volume as available on all nodes',
 992         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 993         optional => 1,
 994         default => 0,
 995     }
 996 );
 997
 998 my %iothread_fmt = ( iothread => {
 999         type => 'boolean',
1000         description => "Whether to use iothreads for this drive",
1001         optional => 1,
1002 });
1003
1004 my %model_fmt = (
1005     model => {
1006         type => 'string',
1007         format => 'urlencoded',
1008         format_description => 'model',
1009         maxLength => 40*3, # *3 since it's %xx url enoded
1010         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1011         optional => 1,
1012     },
1013 );
1014
1015 my %queues_fmt = (
1016     queues => {
1017         type => 'integer',
1018         description => "Number of queues.",
1019         minimum => 2,
1020         optional => 1
1021     }
1022 );
1023
1024 my %scsiblock_fmt = (
1025     scsiblock => {
1026         type => 'boolean',
1027         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1028         optional => 1,
1029         default => 0,
1030     },
1031 );
1032
1033 my %ssd_fmt = (
1034     ssd => {
1035         type => 'boolean',
1036         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1037         optional => 1,
1038     },
1039 );
1040
1041 my $add_throttle_desc = sub {
1042     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1043     my $d = {
1044         type => $type,
1045         format_description => $unit,
1046         description => "Maximum $what in $longunit.",
1047         optional => 1,
1048     };
1049     $d->{minimum} = $minimum if defined($minimum);
1050     $drivedesc_base{$key} = $d;
1051 };
1052 # throughput: (leaky bucket)
1053 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1054 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1056 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1057 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1059 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1060 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1061 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1062
1063 # pools: (pool of IO before throttling starts taking effect)
1064 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1065 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1067 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1068 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1069 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1070
1071 # burst lengths
1072 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1073 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1075 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1078
1079 # legacy support
1080 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1081 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1082 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1083 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1084
1085 my $ide_fmt = {
1086     %drivedesc_base,
1087     %model_fmt,
1088     %ssd_fmt,
1089 };
1090 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1091
1092 my $idedesc = {
1093     optional => 1,
1094     type => 'string', format => $ide_fmt,
1095     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1096 };
1097 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1098
1099 my $scsi_fmt = {
1100     %drivedesc_base,
1101     %iothread_fmt,
1102     %queues_fmt,
1103     %scsiblock_fmt,
1104     %ssd_fmt,
1105 };
1106 my $scsidesc = {
1107     optional => 1,
1108     type => 'string', format => $scsi_fmt,
1109     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1112
1113 my $sata_fmt = {
1114     %drivedesc_base,
1115     %ssd_fmt,
1116 };
1117 my $satadesc = {
1118     optional => 1,
1119     type => 'string', format => $sata_fmt,
1120     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1123
1124 my $virtio_fmt = {
1125     %drivedesc_base,
1126     %iothread_fmt,
1127 };
1128 my $virtiodesc = {
1129     optional => 1,
1130     type => 'string', format => $virtio_fmt,
1131     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1132 };
1133 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1134
1135 my $alldrive_fmt = {
1136     %drivedesc_base,
1137     %iothread_fmt,
1138     %model_fmt,
1139     %queues_fmt,
1140     %scsiblock_fmt,
1141     %ssd_fmt,
1142 };
1143
1144 my $efidisk_fmt = {
1145     volume => { alias => 'file' },
1146     file => {
1147         type => 'string',
1148         format => 'pve-volume-id-or-qm-path',
1149         default_key => 1,
1150         format_description => 'volume',
1151         description => "The drive's backing volume.",
1152     },
1153     format => get_standard_option('pve-qm-image-format'),
1154     size => {
1155         type => 'string',
1156         format => 'disk-size',
1157         format_description => 'DiskSize',
1158         description => "Disk size. This is purely informational and has no effect.",
1159         optional => 1,
1160     },
1161 };
1162
1163 my $efidisk_desc = {
1164     optional => 1,
1165     type => 'string', format => $efidisk_fmt,
1166     description => "Configure a Disk for storing EFI vars",
1167 };
1168
1169 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1170
1171 my $usb_fmt = {
1172     host => {
1173         default_key => 1,
1174         type => 'string', format => 'pve-qm-usb-device',
1175         format_description => 'HOSTUSBDEVICE|spice',
1176         description => <<EODESCR,
1177 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1178
1179  'bus-port(.port)*' (decimal numbers) or
1180  'vendor_id:product_id' (hexadeciaml numbers) or
1181  'spice'
1182
1183 You can use the 'lsusb -t' command to list existing usb devices.
1184
1185 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1186
1187 The value 'spice' can be used to add a usb redirection devices for spice.
1188 EODESCR
1189     },
1190     usb3 => {
1191         optional => 1,
1192         type => 'boolean',
1193         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1194         default => 0,
1195     },
1196 };
1197
1198 my $usbdesc = {
1199     optional => 1,
1200     type => 'string', format => $usb_fmt,
1201     description => "Configure an USB device (n is 0 to 4).",
1202 };
1203 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1204
1205 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1206 my $hostpci_fmt = {
1207     host => {
1208         default_key => 1,
1209         type => 'string',
1210         pattern => qr/$PCIRE(;$PCIRE)*/,
1211         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1212         description => <<EODESCR,
1213 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1214 of PCI virtual functions of the host. HOSTPCIID syntax is:
1215
1216 'bus:dev.func' (hexadecimal numbers)
1217
1218 You can us the 'lspci' command to list existing PCI devices.
1219 EODESCR
1220     },
1221     rombar => {
1222         type => 'boolean',
1223         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1224         optional => 1,
1225         default => 1,
1226     },
1227     romfile => {
1228         type => 'string',
1229         pattern => '[^,;]+',
1230         format_description => 'string',
1231         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1232         optional => 1,
1233     },
1234     pcie => {
1235         type => 'boolean',
1236         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1237         optional => 1,
1238         default => 0,
1239     },
1240     'x-vga' => {
1241         type => 'boolean',
1242         description =>  "Enable vfio-vga device support.",
1243         optional => 1,
1244         default => 0,
1245     },
1246     'mdev' => {
1247         type => 'string',
1248         format_description => 'string',
1249         pattern => '[^/\.:]+',
1250         optional => 1,
1251         description => <<EODESCR
1252 The type of mediated device to use.
1253 An instance of this type will be created on startup of the VM and
1254 will be cleaned up when the VM stops.
1255 EODESCR
1256     }
1257 };
1258 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1259
1260 my $hostpcidesc = {
1261         optional => 1,
1262         type => 'string', format => 'pve-qm-hostpci',
1263         description => "Map host PCI devices into guest.",
1264         verbose_description =>  <<EODESCR,
1265 Map host PCI devices into guest.
1266
1267 NOTE: This option allows direct access to host hardware. So it is no longer
1268 possible to migrate such machines - use with special care.
1269
1270 CAUTION: Experimental! User reported problems with this option.
1271 EODESCR
1272 };
1273 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1274
1275 my $serialdesc = {
1276         optional => 1,
1277         type => 'string',
1278         pattern => '(/dev/.+|socket)',
1279         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1280         verbose_description =>  <<EODESCR,
1281 Create a serial device inside the VM (n is 0 to 3), and pass through a
1282 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1283 host side (use 'qm terminal' to open a terminal connection).
1284
1285 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1286
1287 CAUTION: Experimental! User reported problems with this option.
1288 EODESCR
1289 };
1290
1291 my $paralleldesc= {
1292         optional => 1,
1293         type => 'string',
1294         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1295         description =>  "Map host parallel devices (n is 0 to 2).",
1296         verbose_description =>  <<EODESCR,
1297 Map host parallel devices (n is 0 to 2).
1298
1299 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1300
1301 CAUTION: Experimental! User reported problems with this option.
1302 EODESCR
1303 };
1304
1305 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1306     $confdesc->{"parallel$i"} = $paralleldesc;
1307 }
1308
1309 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1310     $confdesc->{"serial$i"} = $serialdesc;
1311 }
1312
1313 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1314     $confdesc->{"hostpci$i"} = $hostpcidesc;
1315 }
1316
1317 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1318     $drivename_hash->{"ide$i"} = 1;
1319     $confdesc->{"ide$i"} = $idedesc;
1320 }
1321
1322 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1323     $drivename_hash->{"sata$i"} = 1;
1324     $confdesc->{"sata$i"} = $satadesc;
1325 }
1326
1327 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1328     $drivename_hash->{"scsi$i"} = 1;
1329     $confdesc->{"scsi$i"} = $scsidesc ;
1330 }
1331
1332 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1333     $drivename_hash->{"virtio$i"} = 1;
1334     $confdesc->{"virtio$i"} = $virtiodesc;
1335 }
1336
1337 $drivename_hash->{efidisk0} = 1;
1338 $confdesc->{efidisk0} = $efidisk_desc;
1339
1340 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1341     $confdesc->{"usb$i"} = $usbdesc;
1342 }
1343
1344 my $unuseddesc = {
1345     optional => 1,
1346     type => 'string', format => 'pve-volume-id',
1347     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1348 };
1349
1350 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1351     $confdesc->{"unused$i"} = $unuseddesc;
1352 }
1353
1354 my $kvm_api_version = 0;
1355
1356 sub kvm_version {
1357     return $kvm_api_version if $kvm_api_version;
1358
1359     open my $fh, '<', '/dev/kvm'
1360         or return undef;
1361
1362     # 0xae00 => KVM_GET_API_VERSION
1363     $kvm_api_version = ioctl($fh, 0xae00, 0);
1364
1365     return $kvm_api_version;
1366 }
1367
1368 my $kvm_user_version;
1369
1370 sub kvm_user_version {
1371
1372     return $kvm_user_version if $kvm_user_version;
1373
1374     $kvm_user_version = 'unknown';
1375
1376     my $code = sub {
1377         my $line = shift;
1378         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1379             $kvm_user_version = $2;
1380         }
1381     };
1382
1383     eval { run_command("kvm -version", outfunc => $code); };
1384     warn $@ if $@;
1385
1386     return $kvm_user_version;
1387
1388 }
1389
1390 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1391
1392 sub valid_drive_names {
1393     # order is important - used to autoselect boot disk
1394     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1395             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1396             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1397             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1398             'efidisk0');
1399 }
1400
1401 sub is_valid_drivename {
1402     my $dev = shift;
1403
1404     return defined($drivename_hash->{$dev});
1405 }
1406
1407 sub option_exists {
1408     my $key = shift;
1409     return defined($confdesc->{$key});
1410 }
1411
1412 sub nic_models {
1413     return $nic_model_list;
1414 }
1415
1416 sub os_list_description {
1417
1418     return {
1419         other => 'Other',
1420         wxp => 'Windows XP',
1421         w2k => 'Windows 2000',
1422         w2k3 =>, 'Windows 2003',
1423         w2k8 => 'Windows 2008',
1424         wvista => 'Windows Vista',
1425         win7 => 'Windows 7',
1426         win8 => 'Windows 8/2012',
1427         win10 => 'Windows 10/2016',
1428         l24 => 'Linux 2.4',
1429         l26 => 'Linux 2.6',
1430     };
1431 }
1432
1433 my $cdrom_path;
1434
1435 sub get_cdrom_path {
1436
1437     return  $cdrom_path if $cdrom_path;
1438
1439     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1440     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1441     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1442 }
1443
1444 sub get_iso_path {
1445     my ($storecfg, $vmid, $cdrom) = @_;
1446
1447     if ($cdrom eq 'cdrom') {
1448         return get_cdrom_path();
1449     } elsif ($cdrom eq 'none') {
1450         return '';
1451     } elsif ($cdrom =~ m|^/|) {
1452         return $cdrom;
1453     } else {
1454         return PVE::Storage::path($storecfg, $cdrom);
1455     }
1456 }
1457
1458 # try to convert old style file names to volume IDs
1459 sub filename_to_volume_id {
1460     my ($vmid, $file, $media) = @_;
1461
1462      if (!($file eq 'none' || $file eq 'cdrom' ||
1463           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1464
1465         return undef if $file =~ m|/|;
1466
1467         if ($media && $media eq 'cdrom') {
1468             $file = "local:iso/$file";
1469         } else {
1470             $file = "local:$vmid/$file";
1471         }
1472     }
1473
1474     return $file;
1475 }
1476
1477 sub verify_media_type {
1478     my ($opt, $vtype, $media) = @_;
1479
1480     return if !$media;
1481
1482     my $etype;
1483     if ($media eq 'disk') {
1484         $etype = 'images';
1485     } elsif ($media eq 'cdrom') {
1486         $etype = 'iso';
1487     } else {
1488         die "internal error";
1489     }
1490
1491     return if ($vtype eq $etype);
1492
1493     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1494 }
1495
1496 sub cleanup_drive_path {
1497     my ($opt, $storecfg, $drive) = @_;
1498
1499     # try to convert filesystem paths to volume IDs
1500
1501     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1502         ($drive->{file} !~ m|^/dev/.+|) &&
1503         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1504         ($drive->{file} !~ m/^\d+$/)) {
1505         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1506         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1507         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1508         verify_media_type($opt, $vtype, $drive->{media});
1509         $drive->{file} = $volid;
1510     }
1511
1512     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1513 }
1514
1515 sub parse_hotplug_features {
1516     my ($data) = @_;
1517
1518     my $res = {};
1519
1520     return $res if $data eq '0';
1521
1522     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1523
1524     foreach my $feature (PVE::Tools::split_list($data)) {
1525         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1526             $res->{$1} = 1;
1527         } else {
1528             die "invalid hotplug feature '$feature'\n";
1529         }
1530     }
1531     return $res;
1532 }
1533
1534 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1535 sub pve_verify_hotplug_features {
1536     my ($value, $noerr) = @_;
1537
1538     return $value if parse_hotplug_features($value);
1539
1540     return undef if $noerr;
1541
1542     die "unable to parse hotplug option\n";
1543 }
1544
1545 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1546 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1547 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1548 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1549 #        [,iothread=on][,serial=serial][,model=model]
1550
1551 sub parse_drive {
1552     my ($key, $data) = @_;
1553
1554     my ($interface, $index);
1555
1556     if ($key =~ m/^([^\d]+)(\d+)$/) {
1557         $interface = $1;
1558         $index = $2;
1559     } else {
1560         return undef;
1561     }
1562
1563     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1564                                      : $confdesc->{$key}->{format};
1565     if (!$desc) {
1566         warn "invalid drive key: $key\n";
1567         return undef;
1568     }
1569     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1570     return undef if !$res;
1571     $res->{interface} = $interface;
1572     $res->{index} = $index;
1573
1574     my $error = 0;
1575     foreach my $opt (qw(bps bps_rd bps_wr)) {
1576         if (my $bps = defined(delete $res->{$opt})) {
1577             if (defined($res->{"m$opt"})) {
1578                 warn "both $opt and m$opt specified\n";
1579                 ++$error;
1580                 next;
1581             }
1582             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1583         }
1584     }
1585
1586     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1587     for my $requirement (
1588         [mbps_max => 'mbps'],
1589         [mbps_rd_max => 'mbps_rd'],
1590         [mbps_wr_max => 'mbps_wr'],
1591         [miops_max => 'miops'],
1592         [miops_rd_max => 'miops_rd'],
1593         [miops_wr_max => 'miops_wr'],
1594         [bps_max_length => 'mbps_max'],
1595         [bps_rd_max_length => 'mbps_rd_max'],
1596         [bps_wr_max_length => 'mbps_wr_max'],
1597         [iops_max_length => 'iops_max'],
1598         [iops_rd_max_length => 'iops_rd_max'],
1599         [iops_wr_max_length => 'iops_wr_max']) {
1600         my ($option, $requires) = @$requirement;
1601         if ($res->{$option} && !$res->{$requires}) {
1602             warn "$option requires $requires\n";
1603             ++$error;
1604         }
1605     }
1606
1607     return undef if $error;
1608
1609     return undef if $res->{mbps_rd} && $res->{mbps};
1610     return undef if $res->{mbps_wr} && $res->{mbps};
1611     return undef if $res->{iops_rd} && $res->{iops};
1612     return undef if $res->{iops_wr} && $res->{iops};
1613
1614     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1615         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1616         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1617         return undef if $res->{interface} eq 'virtio';
1618     }
1619
1620     if (my $size = $res->{size}) {
1621         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1622     }
1623
1624     return $res;
1625 }
1626
1627 sub print_drive {
1628     my ($vmid, $drive) = @_;
1629     my $data = { %$drive };
1630     delete $data->{$_} for qw(index interface);
1631     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1632 }
1633
1634 sub scsi_inquiry {
1635     my($fh, $noerr) = @_;
1636
1637     my $SG_IO = 0x2285;
1638     my $SG_GET_VERSION_NUM = 0x2282;
1639
1640     my $versionbuf = "\x00" x 8;
1641     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1642     if (!$ret) {
1643         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1644         return undef;
1645     }
1646     my $version = unpack("I", $versionbuf);
1647     if ($version < 30000) {
1648         die "scsi generic interface too old\n"  if !$noerr;
1649         return undef;
1650     }
1651
1652     my $buf = "\x00" x 36;
1653     my $sensebuf = "\x00" x 8;
1654     my $cmd = pack("C x3 C x1", 0x12, 36);
1655
1656     # see /usr/include/scsi/sg.h
1657     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1658
1659     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1660                       length($sensebuf), 0, length($buf), $buf,
1661                       $cmd, $sensebuf, 6000);
1662
1663     $ret = ioctl($fh, $SG_IO, $packet);
1664     if (!$ret) {
1665         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1666         return undef;
1667     }
1668
1669     my @res = unpack($sg_io_hdr_t, $packet);
1670     if ($res[17] || $res[18]) {
1671         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1672         return undef;
1673     }
1674
1675     my $res = {};
1676     (my $byte0, my $byte1, $res->{vendor},
1677      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1678
1679     $res->{removable} = $byte1 & 128 ? 1 : 0;
1680     $res->{type} = $byte0 & 31;
1681
1682     return $res;
1683 }
1684
1685 sub path_is_scsi {
1686     my ($path) = @_;
1687
1688     my $fh = IO::File->new("+<$path") || return undef;
1689     my $res = scsi_inquiry($fh, 1);
1690     close($fh);
1691
1692     return $res;
1693 }
1694
1695 sub machine_type_is_q35 {
1696     my ($conf) = @_;
1697
1698     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1699 }
1700
1701 sub print_tabletdevice_full {
1702     my ($conf, $arch) = @_;
1703
1704     my $q35 = machine_type_is_q35($conf);
1705
1706     # we use uhci for old VMs because tablet driver was buggy in older qemu
1707     my $usbbus;
1708     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1709         $usbbus = 'ehci';
1710     } else {
1711         $usbbus = 'uhci';
1712     }
1713
1714     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1715 }
1716
1717 sub print_keyboarddevice_full {
1718     my ($conf, $arch, $machine) = @_;
1719
1720     return undef if $arch ne 'aarch64';
1721
1722     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1723 }
1724
1725 sub print_drivedevice_full {
1726     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1727
1728     my $device = '';
1729     my $maxdev = 0;
1730
1731     if ($drive->{interface} eq 'virtio') {
1732         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1733         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1734         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1735     } elsif ($drive->{interface} eq 'scsi') {
1736
1737         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1738         my $unit = $drive->{index} % $maxdev;
1739         my $devicetype = 'hd';
1740         my $path = '';
1741         if (drive_is_cdrom($drive)) {
1742             $devicetype = 'cd';
1743         } else {
1744             if ($drive->{file} =~ m|^/|) {
1745                 $path = $drive->{file};
1746                 if (my $info = path_is_scsi($path)) {
1747                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1748                         $devicetype = 'block';
1749                     } elsif ($info->{type} == 1) { # tape
1750                         $devicetype = 'generic';
1751                     }
1752                 }
1753             } else {
1754                  $path = PVE::Storage::path($storecfg, $drive->{file});
1755             }
1756
1757             if($path =~ m/^iscsi\:\/\//){
1758                 $devicetype = 'generic';
1759             }
1760         }
1761
1762         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1763             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1764         } else {
1765             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1766         }
1767
1768         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1769             $device .= ",rotation_rate=1";
1770         }
1771
1772     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1773         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1774         my $controller = int($drive->{index} / $maxdev);
1775         my $unit = $drive->{index} % $maxdev;
1776         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1777
1778         $device = "ide-$devicetype";
1779         if ($drive->{interface} eq 'ide') {
1780             $device .= ",bus=ide.$controller,unit=$unit";
1781         } else {
1782             $device .= ",bus=ahci$controller.$unit";
1783         }
1784         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1785
1786         if ($devicetype eq 'hd') {
1787             if (my $model = $drive->{model}) {
1788                 $model = URI::Escape::uri_unescape($model);
1789                 $device .= ",model=$model";
1790             }
1791             if ($drive->{ssd}) {
1792                 $device .= ",rotation_rate=1";
1793             }
1794         }
1795     } elsif ($drive->{interface} eq 'usb') {
1796         die "implement me";
1797         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1798     } else {
1799         die "unsupported interface type";
1800     }
1801
1802     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1803
1804     if (my $serial = $drive->{serial}) {
1805         $serial = URI::Escape::uri_unescape($serial);
1806         $device .= ",serial=$serial";
1807     }
1808
1809
1810     return $device;
1811 }
1812
1813 sub get_initiator_name {
1814     my $initiator;
1815
1816     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1817     while (defined(my $line = <$fh>)) {
1818         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1819         $initiator = $1;
1820         last;
1821     }
1822     $fh->close();
1823
1824     return $initiator;
1825 }
1826
1827 sub print_drive_full {
1828     my ($storecfg, $vmid, $drive) = @_;
1829
1830     my $path;
1831     my $volid = $drive->{file};
1832     my $format;
1833
1834     if (drive_is_cdrom($drive)) {
1835         $path = get_iso_path($storecfg, $vmid, $volid);
1836     } else {
1837         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1838         if ($storeid) {
1839             $path = PVE::Storage::path($storecfg, $volid);
1840             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1841             $format = qemu_img_format($scfg, $volname);
1842         } else {
1843             $path = $volid;
1844             $format = "raw";
1845         }
1846    }
1847
1848     my $opts = '';
1849     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1850     foreach my $o (@qemu_drive_options) {
1851         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1852     }
1853
1854     # snapshot only accepts on|off
1855     if (defined($drive->{snapshot})) {
1856         my $v = $drive->{snapshot} ? 'on' : 'off';
1857         $opts .= ",snapshot=$v";
1858     }
1859
1860     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1861         my ($dir, $qmpname) = @$type;
1862         if (my $v = $drive->{"mbps$dir"}) {
1863             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1864         }
1865         if (my $v = $drive->{"mbps${dir}_max"}) {
1866             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1867         }
1868         if (my $v = $drive->{"bps${dir}_max_length"}) {
1869             $opts .= ",throttling.bps$qmpname-max-length=$v";
1870         }
1871         if (my $v = $drive->{"iops${dir}"}) {
1872             $opts .= ",throttling.iops$qmpname=$v";
1873         }
1874         if (my $v = $drive->{"iops${dir}_max"}) {
1875             $opts .= ",throttling.iops$qmpname-max=$v";
1876         }
1877         if (my $v = $drive->{"iops${dir}_max_length"}) {
1878             $opts .= ",throttling.iops$qmpname-max-length=$v";
1879         }
1880     }
1881
1882     $opts .= ",format=$format" if $format && !$drive->{format};
1883
1884     my $cache_direct = 0;
1885
1886     if (my $cache = $drive->{cache}) {
1887         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1888     } elsif (!drive_is_cdrom($drive)) {
1889         $opts .= ",cache=none";
1890         $cache_direct = 1;
1891     }
1892
1893     # aio native works only with O_DIRECT
1894     if (!$drive->{aio}) {
1895         if($cache_direct) {
1896             $opts .= ",aio=native";
1897         } else {
1898             $opts .= ",aio=threads";
1899         }
1900     }
1901
1902     if (!drive_is_cdrom($drive)) {
1903         my $detectzeroes;
1904         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1905             $detectzeroes = 'off';
1906         } elsif ($drive->{discard}) {
1907             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1908         } else {
1909             # This used to be our default with discard not being specified:
1910             $detectzeroes = 'on';
1911         }
1912         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1913     }
1914
1915     my $pathinfo = $path ? "file=$path," : '';
1916
1917     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1918 }
1919
1920 sub print_netdevice_full {
1921     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1922
1923     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1924
1925     my $device = $net->{model};
1926     if ($net->{model} eq 'virtio') {
1927          $device = 'virtio-net-pci';
1928      };
1929
1930     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1931     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1932     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1933         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1934         my $vectors = $net->{queues} * 2 + 2;
1935         $tmpstr .= ",vectors=$vectors,mq=on";
1936     }
1937     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1938
1939     if ($use_old_bios_files) {
1940         my $romfile;
1941         if ($device eq 'virtio-net-pci') {
1942             $romfile = 'pxe-virtio.rom';
1943         } elsif ($device eq 'e1000') {
1944             $romfile = 'pxe-e1000.rom';
1945         } elsif ($device eq 'ne2k') {
1946             $romfile = 'pxe-ne2k_pci.rom';
1947         } elsif ($device eq 'pcnet') {
1948             $romfile = 'pxe-pcnet.rom';
1949         } elsif ($device eq 'rtl8139') {
1950             $romfile = 'pxe-rtl8139.rom';
1951         }
1952         $tmpstr .= ",romfile=$romfile" if $romfile;
1953     }
1954
1955     return $tmpstr;
1956 }
1957
1958 sub print_netdev_full {
1959     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1960
1961     my $i = '';
1962     if ($netid =~ m/^net(\d+)$/) {
1963         $i = int($1);
1964     }
1965
1966     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1967
1968     my $ifname = "tap${vmid}i$i";
1969
1970     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1971     die "interface name '$ifname' is too long (max 15 character)\n"
1972         if length($ifname) >= 16;
1973
1974     my $vhostparam = '';
1975     if (is_native($arch)) {
1976         $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1977     }
1978
1979     my $vmname = $conf->{name} || "vm$vmid";
1980
1981     my $netdev = "";
1982     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1983
1984     if ($net->{bridge}) {
1985         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1986     } else {
1987         $netdev = "type=user,id=$netid,hostname=$vmname";
1988     }
1989
1990     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1991
1992     return $netdev;
1993 }
1994
1995
1996 sub print_cpu_device {
1997     my ($conf, $id) = @_;
1998
1999     my $kvm = $conf->{kvm} // 1;
2000     my $cpu = $kvm ? "kvm64" : "qemu64";
2001     if (my $cputype = $conf->{cpu}) {
2002         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2003             or die "Cannot parse cpu description: $cputype\n";
2004         $cpu = $cpuconf->{cputype};
2005     }
2006
2007     my $cores = $conf->{cores} || 1;
2008
2009     my $current_core = ($id - 1) % $cores;
2010     my $current_socket = int(($id - 1 - $current_core)/$cores);
2011
2012     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2013 }
2014
2015 my $vga_map = {
2016     'cirrus' => 'cirrus-vga',
2017     'std' => 'VGA',
2018     'vmware' => 'vmware-svga',
2019     'virtio' => 'virtio-vga',
2020 };
2021
2022 sub print_vga_device {
2023     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2024
2025     my $type = $vga_map->{$vga->{type}};
2026     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2027         $type = 'virtio-gpu';
2028     }
2029     my $vgamem_mb = $vga->{memory};
2030     if ($qxlnum) {
2031         $type = $id ? 'qxl' : 'qxl-vga';
2032     }
2033     die "no devicetype for $vga->{type}\n" if !$type;
2034
2035     my $memory = "";
2036     if ($vgamem_mb) {
2037         if ($vga->{type} eq 'virtio') {
2038             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2039             $memory = ",max_hostmem=$bytes";
2040         } elsif ($qxlnum) {
2041             # from https://www.spice-space.org/multiple-monitors.html
2042             $memory = ",vgamem_mb=$vga->{memory}";
2043             my $ram = $vgamem_mb * 4;
2044             my $vram = $vgamem_mb * 2;
2045             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2046         } else {
2047             $memory = ",vgamem_mb=$vga->{memory}";
2048         }
2049     } elsif ($qxlnum && $id) {
2050         $memory = ",ram_size=67108864,vram_size=33554432";
2051     }
2052
2053     my $q35 = machine_type_is_q35($conf);
2054     my $vgaid = "vga" . ($id // '');
2055     my $pciaddr;
2056
2057     if ($q35 && $vgaid eq 'vga') {
2058         # the first display uses pcie.0 bus on q35 machines
2059         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2060     } else {
2061         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2062     }
2063
2064     return "$type,id=${vgaid}${memory}${pciaddr}";
2065 }
2066
2067 sub drive_is_cloudinit {
2068     my ($drive) = @_;
2069     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2070 }
2071
2072 sub drive_is_cdrom {
2073     my ($drive, $exclude_cloudinit) = @_;
2074
2075     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2076
2077     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2078
2079 }
2080
2081 sub parse_number_sets {
2082     my ($set) = @_;
2083     my $res = [];
2084     foreach my $part (split(/;/, $set)) {
2085         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2086             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2087             push @$res, [ $1, $2 ];
2088         } else {
2089             die "invalid range: $part\n";
2090         }
2091     }
2092     return $res;
2093 }
2094
2095 sub parse_numa {
2096     my ($data) = @_;
2097
2098     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2099     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2100     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2101     return $res;
2102 }
2103
2104 sub parse_hostpci {
2105     my ($value) = @_;
2106
2107     return undef if !$value;
2108
2109     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2110
2111     my @idlist = split(/;/, $res->{host});
2112     delete $res->{host};
2113     foreach my $id (@idlist) {
2114         if ($id =~ m/\./) { # full id 00:00.1
2115             push @{$res->{pciid}}, {
2116                 id => $id,
2117             };
2118         } else { # partial id 00:00
2119             $res->{pciid} = PVE::SysFSTools::lspci($id);
2120         }
2121     }
2122     return $res;
2123 }
2124
2125 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2126 sub parse_net {
2127     my ($data) = @_;
2128
2129     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2130     if ($@) {
2131         warn $@;
2132         return undef;
2133     }
2134     if (!defined($res->{macaddr})) {
2135         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2136         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2137     }
2138     return $res;
2139 }
2140
2141 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2142 sub parse_ipconfig {
2143     my ($data) = @_;
2144
2145     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2146     if ($@) {
2147         warn $@;
2148         return undef;
2149     }
2150
2151     if ($res->{gw} && !$res->{ip}) {
2152         warn 'gateway specified without specifying an IP address';
2153         return undef;
2154     }
2155     if ($res->{gw6} && !$res->{ip6}) {
2156         warn 'IPv6 gateway specified without specifying an IPv6 address';
2157         return undef;
2158     }
2159     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2160         warn 'gateway specified together with DHCP';
2161         return undef;
2162     }
2163     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2164         # gw6 + auto/dhcp
2165         warn "IPv6 gateway specified together with $res->{ip6} address";
2166         return undef;
2167     }
2168
2169     if (!$res->{ip} && !$res->{ip6}) {
2170         return { ip => 'dhcp', ip6 => 'dhcp' };
2171     }
2172
2173     return $res;
2174 }
2175
2176 sub print_net {
2177     my $net = shift;
2178
2179     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2180 }
2181
2182 sub add_random_macs {
2183     my ($settings) = @_;
2184
2185     foreach my $opt (keys %$settings) {
2186         next if $opt !~ m/^net(\d+)$/;
2187         my $net = parse_net($settings->{$opt});
2188         next if !$net;
2189         $settings->{$opt} = print_net($net);
2190     }
2191 }
2192
2193 sub vm_is_volid_owner {
2194     my ($storecfg, $vmid, $volid) = @_;
2195
2196     if ($volid !~  m|^/|) {
2197         my ($path, $owner);
2198         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2199         if ($owner && ($owner == $vmid)) {
2200             return 1;
2201         }
2202     }
2203
2204     return undef;
2205 }
2206
2207 sub split_flagged_list {
2208     my $text = shift || '';
2209     $text =~ s/[,;]/ /g;
2210     $text =~ s/^\s+//;
2211     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2212 }
2213
2214 sub join_flagged_list {
2215     my ($how, $lst) = @_;
2216     join $how, map { $lst->{$_} . $_ } keys %$lst;
2217 }
2218
2219 sub vmconfig_delete_pending_option {
2220     my ($conf, $key, $force) = @_;
2221
2222     delete $conf->{pending}->{$key};
2223     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2224     $pending_delete_hash->{$key} = $force ? '!' : '';
2225     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2226 }
2227
2228 sub vmconfig_undelete_pending_option {
2229     my ($conf, $key) = @_;
2230
2231     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2232     delete $pending_delete_hash->{$key};
2233
2234     if (%$pending_delete_hash) {
2235         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2236     } else {
2237         delete $conf->{pending}->{delete};
2238     }
2239 }
2240
2241 sub vmconfig_register_unused_drive {
2242     my ($storecfg, $vmid, $conf, $drive) = @_;
2243
2244     if (drive_is_cloudinit($drive)) {
2245         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2246         warn $@ if $@;
2247     } elsif (!drive_is_cdrom($drive)) {
2248         my $volid = $drive->{file};
2249         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2250             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2251         }
2252     }
2253 }
2254
2255 sub vmconfig_cleanup_pending {
2256     my ($conf) = @_;
2257
2258     # remove pending changes when nothing changed
2259     my $changes;
2260     foreach my $opt (keys %{$conf->{pending}}) {
2261         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2262             $changes = 1;
2263             delete $conf->{pending}->{$opt};
2264         }
2265     }
2266
2267     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2268     my $pending_delete_hash = {};
2269     while (my ($opt, $force) = each %$current_delete_hash) {
2270         if (defined($conf->{$opt})) {
2271             $pending_delete_hash->{$opt} = $force;
2272         } else {
2273             $changes = 1;
2274         }
2275     }
2276
2277     if (%$pending_delete_hash) {
2278         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2279     } else {
2280         delete $conf->{pending}->{delete};
2281     }
2282
2283     return $changes;
2284 }
2285
2286 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2287 my $smbios1_fmt = {
2288     uuid => {
2289         type => 'string',
2290         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2291         format_description => 'UUID',
2292         description => "Set SMBIOS1 UUID.",
2293         optional => 1,
2294     },
2295     version => {
2296         type => 'string',
2297         pattern => '\S+',
2298         format_description => 'string',
2299         description => "Set SMBIOS1 version.",
2300         optional => 1,
2301     },
2302     serial => {
2303         type => 'string',
2304         pattern => '\S+',
2305         format_description => 'string',
2306         description => "Set SMBIOS1 serial number.",
2307         optional => 1,
2308     },
2309     manufacturer => {
2310         type => 'string',
2311         pattern => '\S+',
2312         format_description => 'string',
2313         description => "Set SMBIOS1 manufacturer.",
2314         optional => 1,
2315     },
2316     product => {
2317         type => 'string',
2318         pattern => '\S+',
2319         format_description => 'string',
2320         description => "Set SMBIOS1 product ID.",
2321         optional => 1,
2322     },
2323     sku => {
2324         type => 'string',
2325         pattern => '\S+',
2326         format_description => 'string',
2327         description => "Set SMBIOS1 SKU string.",
2328         optional => 1,
2329     },
2330     family => {
2331         type => 'string',
2332         pattern => '\S+',
2333         format_description => 'string',
2334         description => "Set SMBIOS1 family string.",
2335         optional => 1,
2336     },
2337 };
2338
2339 sub parse_smbios1 {
2340     my ($data) = @_;
2341
2342     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2343     warn $@ if $@;
2344     return $res;
2345 }
2346
2347 sub print_smbios1 {
2348     my ($smbios1) = @_;
2349     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2350 }
2351
2352 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2353
2354 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2355 sub verify_bootdisk {
2356     my ($value, $noerr) = @_;
2357
2358     return $value if is_valid_drivename($value);
2359
2360     return undef if $noerr;
2361
2362     die "invalid boot disk '$value'\n";
2363 }
2364
2365 sub parse_watchdog {
2366     my ($value) = @_;
2367
2368     return undef if !$value;
2369
2370     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2371     warn $@ if $@;
2372     return $res;
2373 }
2374
2375 sub parse_guest_agent {
2376     my ($value) = @_;
2377
2378     return {} if !defined($value->{agent});
2379
2380     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2381     warn $@ if $@;
2382
2383     # if the agent is disabled ignore the other potentially set properties
2384     return {} if !$res->{enabled};
2385     return $res;
2386 }
2387
2388 sub parse_vga {
2389     my ($value) = @_;
2390
2391     return {} if !$value;
2392     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2393     warn $@ if $@;
2394     return $res;
2395 }
2396
2397 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2398 sub verify_usb_device {
2399     my ($value, $noerr) = @_;
2400
2401     return $value if parse_usb_device($value);
2402
2403     return undef if $noerr;
2404
2405     die "unable to parse usb device\n";
2406 }
2407
2408 # add JSON properties for create and set function
2409 sub json_config_properties {
2410     my $prop = shift;
2411
2412     foreach my $opt (keys %$confdesc) {
2413         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2414         $prop->{$opt} = $confdesc->{$opt};
2415     }
2416
2417     return $prop;
2418 }
2419
2420 # return copy of $confdesc_cloudinit to generate documentation
2421 sub cloudinit_config_properties {
2422
2423     return dclone($confdesc_cloudinit);
2424 }
2425
2426 sub check_type {
2427     my ($key, $value) = @_;
2428
2429     die "unknown setting '$key'\n" if !$confdesc->{$key};
2430
2431     my $type = $confdesc->{$key}->{type};
2432
2433     if (!defined($value)) {
2434         die "got undefined value\n";
2435     }
2436
2437     if ($value =~ m/[\n\r]/) {
2438         die "property contains a line feed\n";
2439     }
2440
2441     if ($type eq 'boolean') {
2442         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2443         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2444         die "type check ('boolean') failed - got '$value'\n";
2445     } elsif ($type eq 'integer') {
2446         return int($1) if $value =~ m/^(\d+)$/;
2447         die "type check ('integer') failed - got '$value'\n";
2448     } elsif ($type eq 'number') {
2449         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2450         die "type check ('number') failed - got '$value'\n";
2451     } elsif ($type eq 'string') {
2452         if (my $fmt = $confdesc->{$key}->{format}) {
2453             PVE::JSONSchema::check_format($fmt, $value);
2454             return $value;
2455         }
2456         $value =~ s/^\"(.*)\"$/$1/;
2457         return $value;
2458     } else {
2459         die "internal error"
2460     }
2461 }
2462
2463 sub touch_config {
2464     my ($vmid) = @_;
2465
2466     my $conf = PVE::QemuConfig->config_file($vmid);
2467     utime undef, undef, $conf;
2468 }
2469
2470 sub destroy_vm {
2471     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2472
2473     my $conffile = PVE::QemuConfig->config_file($vmid);
2474
2475     my $conf = PVE::QemuConfig->load_config($vmid);
2476
2477     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2478
2479     if ($conf->{template}) {
2480         # check if any base image is still used by a linked clone
2481         foreach_drive($conf, sub {
2482                 my ($ds, $drive) = @_;
2483
2484                 return if drive_is_cdrom($drive);
2485
2486                 my $volid = $drive->{file};
2487
2488                 return if !$volid || $volid =~ m|^/|;
2489
2490                 die "base volume '$volid' is still in use by linked cloned\n"
2491                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2492
2493         });
2494     }
2495
2496     # only remove disks owned by this VM
2497     foreach_drive($conf, sub {
2498         my ($ds, $drive) = @_;
2499
2500         return if drive_is_cdrom($drive, 1);
2501
2502         my $volid = $drive->{file};
2503
2504         return if !$volid || $volid =~ m|^/|;
2505
2506         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2507         return if !$path || !$owner || ($owner != $vmid);
2508
2509         eval {
2510             PVE::Storage::vdisk_free($storecfg, $volid);
2511         };
2512         warn "Could not remove disk '$volid', check manually: $@" if $@;
2513
2514     });
2515
2516     if ($keep_empty_config) {
2517         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2518     } else {
2519         unlink $conffile;
2520     }
2521
2522     # also remove unused disk
2523     eval {
2524         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2525
2526         eval {
2527             PVE::Storage::foreach_volid($dl, sub {
2528                 my ($volid, $sid, $volname, $d) = @_;
2529                 PVE::Storage::vdisk_free($storecfg, $volid);
2530             });
2531         };
2532         warn $@ if $@;
2533
2534     };
2535     warn $@ if $@;
2536 }
2537
2538 sub parse_vm_config {
2539     my ($filename, $raw) = @_;
2540
2541     return undef if !defined($raw);
2542
2543     my $res = {
2544         digest => Digest::SHA::sha1_hex($raw),
2545         snapshots => {},
2546         pending => {},
2547     };
2548
2549     $filename =~ m|/qemu-server/(\d+)\.conf$|
2550         || die "got strange filename '$filename'";
2551
2552     my $vmid = $1;
2553
2554     my $conf = $res;
2555     my $descr;
2556     my $section = '';
2557
2558     my @lines = split(/\n/, $raw);
2559     foreach my $line (@lines) {
2560         next if $line =~ m/^\s*$/;
2561
2562         if ($line =~ m/^\[PENDING\]\s*$/i) {
2563             $section = 'pending';
2564             if (defined($descr)) {
2565                 $descr =~ s/\s+$//;
2566                 $conf->{description} = $descr;
2567             }
2568             $descr = undef;
2569             $conf = $res->{$section} = {};
2570             next;
2571
2572         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2573             $section = $1;
2574             if (defined($descr)) {
2575                 $descr =~ s/\s+$//;
2576                 $conf->{description} = $descr;
2577             }
2578             $descr = undef;
2579             $conf = $res->{snapshots}->{$section} = {};
2580             next;
2581         }
2582
2583         if ($line =~ m/^\#(.*)\s*$/) {
2584             $descr = '' if !defined($descr);
2585             $descr .= PVE::Tools::decode_text($1) . "\n";
2586             next;
2587         }
2588
2589         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2590             $descr = '' if !defined($descr);
2591             $descr .= PVE::Tools::decode_text($2);
2592         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2593             $conf->{snapstate} = $1;
2594         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2595             my $key = $1;
2596             my $value = $2;
2597             $conf->{$key} = $value;
2598         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2599             my $value = $1;
2600             if ($section eq 'pending') {
2601                 $conf->{delete} = $value; # we parse this later
2602             } else {
2603                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2604             }
2605         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2606             my $key = $1;
2607             my $value = $2;
2608             eval { $value = check_type($key, $value); };
2609             if ($@) {
2610                 warn "vm $vmid - unable to parse value of '$key' - $@";
2611             } else {
2612                 $key = 'ide2' if $key eq 'cdrom';
2613                 my $fmt = $confdesc->{$key}->{format};
2614                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2615                     my $v = parse_drive($key, $value);
2616                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2617                         $v->{file} = $volid;
2618                         $value = print_drive($vmid, $v);
2619                     } else {
2620                         warn "vm $vmid - unable to parse value of '$key'\n";
2621                         next;
2622                     }
2623                 }
2624
2625                 $conf->{$key} = $value;
2626             }
2627         }
2628     }
2629
2630     if (defined($descr)) {
2631         $descr =~ s/\s+$//;
2632         $conf->{description} = $descr;
2633     }
2634     delete $res->{snapstate}; # just to be sure
2635
2636     return $res;
2637 }
2638
2639 sub write_vm_config {
2640     my ($filename, $conf) = @_;
2641
2642     delete $conf->{snapstate}; # just to be sure
2643
2644     if ($conf->{cdrom}) {
2645         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2646         $conf->{ide2} = $conf->{cdrom};
2647         delete $conf->{cdrom};
2648     }
2649
2650     # we do not use 'smp' any longer
2651     if ($conf->{sockets}) {
2652         delete $conf->{smp};
2653     } elsif ($conf->{smp}) {
2654         $conf->{sockets} = $conf->{smp};
2655         delete $conf->{cores};
2656         delete $conf->{smp};
2657     }
2658
2659     my $used_volids = {};
2660
2661     my $cleanup_config = sub {
2662         my ($cref, $pending, $snapname) = @_;
2663
2664         foreach my $key (keys %$cref) {
2665             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2666                 $key eq 'snapstate' || $key eq 'pending';
2667             my $value = $cref->{$key};
2668             if ($key eq 'delete') {
2669                 die "propertry 'delete' is only allowed in [PENDING]\n"
2670                     if !$pending;
2671                 # fixme: check syntax?
2672                 next;
2673             }
2674             eval { $value = check_type($key, $value); };
2675             die "unable to parse value of '$key' - $@" if $@;
2676
2677             $cref->{$key} = $value;
2678
2679             if (!$snapname && is_valid_drivename($key)) {
2680                 my $drive = parse_drive($key, $value);
2681                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2682             }
2683         }
2684     };
2685
2686     &$cleanup_config($conf);
2687
2688     &$cleanup_config($conf->{pending}, 1);
2689
2690     foreach my $snapname (keys %{$conf->{snapshots}}) {
2691         die "internal error" if $snapname eq 'pending';
2692         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2693     }
2694
2695     # remove 'unusedX' settings if we re-add a volume
2696     foreach my $key (keys %$conf) {
2697         my $value = $conf->{$key};
2698         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2699             delete $conf->{$key};
2700         }
2701     }
2702
2703     my $generate_raw_config = sub {
2704         my ($conf, $pending) = @_;
2705
2706         my $raw = '';
2707
2708         # add description as comment to top of file
2709         if (defined(my $descr = $conf->{description})) {
2710             if ($descr) {
2711                 foreach my $cl (split(/\n/, $descr)) {
2712                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2713                 }
2714             } else {
2715                 $raw .= "#\n" if $pending;
2716             }
2717         }
2718
2719         foreach my $key (sort keys %$conf) {
2720             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2721             $raw .= "$key: $conf->{$key}\n";
2722         }
2723         return $raw;
2724     };
2725
2726     my $raw = &$generate_raw_config($conf);
2727
2728     if (scalar(keys %{$conf->{pending}})){
2729         $raw .= "\n[PENDING]\n";
2730         $raw .= &$generate_raw_config($conf->{pending}, 1);
2731     }
2732
2733     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2734         $raw .= "\n[$snapname]\n";
2735         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2736     }
2737
2738     return $raw;
2739 }
2740
2741 sub load_defaults {
2742
2743     my $res = {};
2744
2745     # we use static defaults from our JSON schema configuration
2746     foreach my $key (keys %$confdesc) {
2747         if (defined(my $default = $confdesc->{$key}->{default})) {
2748             $res->{$key} = $default;
2749         }
2750     }
2751
2752     return $res;
2753 }
2754
2755 sub config_list {
2756     my $vmlist = PVE::Cluster::get_vmlist();
2757     my $res = {};
2758     return $res if !$vmlist || !$vmlist->{ids};
2759     my $ids = $vmlist->{ids};
2760
2761     foreach my $vmid (keys %$ids) {
2762         my $d = $ids->{$vmid};
2763         next if !$d->{node} || $d->{node} ne $nodename;
2764         next if !$d->{type} || $d->{type} ne 'qemu';
2765         $res->{$vmid}->{exists} = 1;
2766     }
2767     return $res;
2768 }
2769
2770 # test if VM uses local resources (to prevent migration)
2771 sub check_local_resources {
2772     my ($conf, $noerr) = @_;
2773
2774     my $loc_res = 0;
2775
2776     $loc_res = 1 if $conf->{hostusb}; # old syntax
2777     $loc_res = 1 if $conf->{hostpci}; # old syntax
2778
2779     foreach my $k (keys %$conf) {
2780         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2781         # sockets are safe: they will recreated be on the target side post-migrate
2782         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2783         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2784     }
2785
2786     die "VM uses local resources\n" if $loc_res && !$noerr;
2787
2788     return $loc_res;
2789 }
2790
2791 # check if used storages are available on all nodes (use by migrate)
2792 sub check_storage_availability {
2793     my ($storecfg, $conf, $node) = @_;
2794
2795     foreach_drive($conf, sub {
2796         my ($ds, $drive) = @_;
2797
2798         my $volid = $drive->{file};
2799         return if !$volid;
2800
2801         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2802         return if !$sid;
2803
2804         # check if storage is available on both nodes
2805         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2806         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2807    });
2808 }
2809
2810 # list nodes where all VM images are available (used by has_feature API)
2811 sub shared_nodes {
2812     my ($conf, $storecfg) = @_;
2813
2814     my $nodelist = PVE::Cluster::get_nodelist();
2815     my $nodehash = { map { $_ => 1 } @$nodelist };
2816     my $nodename = PVE::INotify::nodename();
2817
2818     foreach_drive($conf, sub {
2819         my ($ds, $drive) = @_;
2820
2821         my $volid = $drive->{file};
2822         return if !$volid;
2823
2824         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2825         if ($storeid) {
2826             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2827             if ($scfg->{disable}) {
2828                 $nodehash = {};
2829             } elsif (my $avail = $scfg->{nodes}) {
2830                 foreach my $node (keys %$nodehash) {
2831                     delete $nodehash->{$node} if !$avail->{$node};
2832                 }
2833             } elsif (!$scfg->{shared}) {
2834                 foreach my $node (keys %$nodehash) {
2835                     delete $nodehash->{$node} if $node ne $nodename
2836                 }
2837             }
2838         }
2839     });
2840
2841     return $nodehash
2842 }
2843
2844 sub check_cmdline {
2845     my ($pidfile, $pid) = @_;
2846
2847     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2848     if (defined($fh)) {
2849         my $line = <$fh>;
2850         $fh->close;
2851         return undef if !$line;
2852         my @param = split(/\0/, $line);
2853
2854         my $cmd = $param[0];
2855         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2856
2857         for (my $i = 0; $i < scalar (@param); $i++) {
2858             my $p = $param[$i];
2859             next if !$p;
2860             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2861                 my $p = $param[$i+1];
2862                 return 1 if $p && ($p eq $pidfile);
2863                 return undef;
2864             }
2865         }
2866     }
2867     return undef;
2868 }
2869
2870 sub check_running {
2871     my ($vmid, $nocheck, $node) = @_;
2872
2873     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2874
2875     die "unable to find configuration file for VM $vmid - no such machine\n"
2876         if !$nocheck && ! -f $filename;
2877
2878     my $pidfile = pidfile_name($vmid);
2879
2880     if (my $fd = IO::File->new("<$pidfile")) {
2881         my $st = stat($fd);
2882         my $line = <$fd>;
2883         close($fd);
2884
2885         my $mtime = $st->mtime;
2886         if ($mtime > time()) {
2887             warn "file '$filename' modified in future\n";
2888         }
2889
2890         if ($line =~ m/^(\d+)$/) {
2891             my $pid = $1;
2892             if (check_cmdline($pidfile, $pid)) {
2893                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2894                     return $pid;
2895                 }
2896             }
2897         }
2898     }
2899
2900     return undef;
2901 }
2902
2903 sub vzlist {
2904
2905     my $vzlist = config_list();
2906
2907     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2908
2909     while (defined(my $de = $fd->read)) {
2910         next if $de !~ m/^(\d+)\.pid$/;
2911         my $vmid = $1;
2912         next if !defined($vzlist->{$vmid});
2913         if (my $pid = check_running($vmid)) {
2914             $vzlist->{$vmid}->{pid} = $pid;
2915         }
2916     }
2917
2918     return $vzlist;
2919 }
2920
2921 sub disksize {
2922     my ($storecfg, $conf) = @_;
2923
2924     my $bootdisk = $conf->{bootdisk};
2925     return undef if !$bootdisk;
2926     return undef if !is_valid_drivename($bootdisk);
2927
2928     return undef if !$conf->{$bootdisk};
2929
2930     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2931     return undef if !defined($drive);
2932
2933     return undef if drive_is_cdrom($drive);
2934
2935     my $volid = $drive->{file};
2936     return undef if !$volid;
2937
2938     return $drive->{size};
2939 }
2940
2941 our $vmstatus_return_properties = {
2942     vmid => get_standard_option('pve-vmid'),
2943     status => {
2944         description => "Qemu process status.",
2945         type => 'string',
2946         enum => ['stopped', 'running'],
2947     },
2948     maxmem => {
2949         description => "Maximum memory in bytes.",
2950         type => 'integer',
2951         optional => 1,
2952         renderer => 'bytes',
2953     },
2954     maxdisk => {
2955         description => "Root disk size in bytes.",
2956         type => 'integer',
2957         optional => 1,
2958         renderer => 'bytes',
2959     },
2960     name => {
2961         description => "VM name.",
2962         type => 'string',
2963         optional => 1,
2964     },
2965     qmpstatus => {
2966         description => "Qemu QMP agent status.",
2967         type => 'string',
2968         optional => 1,
2969     },
2970     pid => {
2971         description => "PID of running qemu process.",
2972         type => 'integer',
2973         optional => 1,
2974     },
2975     uptime => {
2976         description => "Uptime.",
2977         type => 'integer',
2978         optional => 1,
2979         renderer => 'duration',
2980     },
2981     cpus => {
2982         description => "Maximum usable CPUs.",
2983         type => 'number',
2984         optional => 1,
2985     },
2986 };
2987
2988 my $last_proc_pid_stat;
2989
2990 # get VM status information
2991 # This must be fast and should not block ($full == false)
2992 # We only query KVM using QMP if $full == true (this can be slow)
2993 sub vmstatus {
2994     my ($opt_vmid, $full) = @_;
2995
2996     my $res = {};
2997
2998     my $storecfg = PVE::Storage::config();
2999
3000     my $list = vzlist();
3001     my $defaults = load_defaults();
3002
3003     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3004
3005     my $cpucount = $cpuinfo->{cpus} || 1;
3006
3007     foreach my $vmid (keys %$list) {
3008         next if $opt_vmid && ($vmid ne $opt_vmid);
3009
3010         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3011         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3012
3013         my $d = { vmid => $vmid };
3014         $d->{pid} = $list->{$vmid}->{pid};
3015
3016         # fixme: better status?
3017         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3018
3019         my $size = disksize($storecfg, $conf);
3020         if (defined($size)) {
3021             $d->{disk} = 0; # no info available
3022             $d->{maxdisk} = $size;
3023         } else {
3024             $d->{disk} = 0;
3025             $d->{maxdisk} = 0;
3026         }
3027
3028         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3029             * ($conf->{cores} || $defaults->{cores});
3030         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3031         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3032
3033         $d->{name} = $conf->{name} || "VM $vmid";
3034         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3035             : $defaults->{memory}*(1024*1024);
3036
3037         if ($conf->{balloon}) {
3038             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3039             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3040                 : $defaults->{shares};
3041         }
3042
3043         $d->{uptime} = 0;
3044         $d->{cpu} = 0;
3045         $d->{mem} = 0;
3046
3047         $d->{netout} = 0;
3048         $d->{netin} = 0;
3049
3050         $d->{diskread} = 0;
3051         $d->{diskwrite} = 0;
3052
3053         $d->{template} = PVE::QemuConfig->is_template($conf);
3054
3055         $d->{serial} = 1 if conf_has_serial($conf);
3056
3057         $res->{$vmid} = $d;
3058     }
3059
3060     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3061     foreach my $dev (keys %$netdev) {
3062         next if $dev !~ m/^tap([1-9]\d*)i/;
3063         my $vmid = $1;
3064         my $d = $res->{$vmid};
3065         next if !$d;
3066
3067         $d->{netout} += $netdev->{$dev}->{receive};
3068         $d->{netin} += $netdev->{$dev}->{transmit};
3069
3070         if ($full) {
3071             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3072             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3073         }
3074
3075     }
3076
3077     my $ctime = gettimeofday;
3078
3079     foreach my $vmid (keys %$list) {
3080
3081         my $d = $res->{$vmid};
3082         my $pid = $d->{pid};
3083         next if !$pid;
3084
3085         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3086         next if !$pstat; # not running
3087
3088         my $used = $pstat->{utime} + $pstat->{stime};
3089
3090         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3091
3092         if ($pstat->{vsize}) {
3093             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3094         }
3095
3096         my $old = $last_proc_pid_stat->{$pid};
3097         if (!$old) {
3098             $last_proc_pid_stat->{$pid} = {
3099                 time => $ctime,
3100                 used => $used,
3101                 cpu => 0,
3102             };
3103             next;
3104         }
3105
3106         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3107
3108         if ($dtime > 1000) {
3109             my $dutime = $used -  $old->{used};
3110
3111             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3112             $last_proc_pid_stat->{$pid} = {
3113                 time => $ctime,
3114                 used => $used,
3115                 cpu => $d->{cpu},
3116             };
3117         } else {
3118             $d->{cpu} = $old->{cpu};
3119         }
3120     }
3121
3122     return $res if !$full;
3123
3124     my $qmpclient = PVE::QMPClient->new();
3125
3126     my $ballooncb = sub {
3127         my ($vmid, $resp) = @_;
3128
3129         my $info = $resp->{'return'};
3130         return if !$info->{max_mem};
3131
3132         my $d = $res->{$vmid};
3133
3134         # use memory assigned to VM
3135         $d->{maxmem} = $info->{max_mem};
3136         $d->{balloon} = $info->{actual};
3137
3138         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3139             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3140             $d->{freemem} = $info->{free_mem};
3141         }
3142
3143         $d->{ballooninfo} = $info;
3144     };
3145
3146     my $blockstatscb = sub {
3147         my ($vmid, $resp) = @_;
3148         my $data = $resp->{'return'} || [];
3149         my $totalrdbytes = 0;
3150         my $totalwrbytes = 0;
3151
3152         for my $blockstat (@$data) {
3153             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3154             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3155
3156             $blockstat->{device} =~ s/drive-//;
3157             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3158         }
3159         $res->{$vmid}->{diskread} = $totalrdbytes;
3160         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3161     };
3162
3163     my $statuscb = sub {
3164         my ($vmid, $resp) = @_;
3165
3166         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3167         # this fails if ballon driver is not loaded, so this must be
3168         # the last commnand (following command are aborted if this fails).
3169         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3170
3171         my $status = 'unknown';
3172         if (!defined($status = $resp->{'return'}->{status})) {
3173             warn "unable to get VM status\n";
3174             return;
3175         }
3176
3177         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3178     };
3179
3180     foreach my $vmid (keys %$list) {
3181         next if $opt_vmid && ($vmid ne $opt_vmid);
3182         next if !$res->{$vmid}->{pid}; # not running
3183         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3184     }
3185
3186     $qmpclient->queue_execute(undef, 2);
3187
3188     foreach my $vmid (keys %$list) {
3189         next if $opt_vmid && ($vmid ne $opt_vmid);
3190         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3191     }
3192
3193     return $res;
3194 }
3195
3196 sub foreach_drive {
3197     my ($conf, $func, @param) = @_;
3198
3199     foreach my $ds (valid_drive_names()) {
3200         next if !defined($conf->{$ds});
3201
3202         my $drive = parse_drive($ds, $conf->{$ds});
3203         next if !$drive;
3204
3205         &$func($ds, $drive, @param);
3206     }
3207 }
3208
3209 sub foreach_volid {
3210     my ($conf, $func, @param) = @_;
3211
3212     my $volhash = {};
3213
3214     my $test_volid = sub {
3215         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3216
3217         return if !$volid;
3218
3219         $volhash->{$volid}->{cdrom} //= 1;
3220         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3221
3222         $volhash->{$volid}->{replicate} //= 0;
3223         $volhash->{$volid}->{replicate} = 1 if $replicate;
3224
3225         $volhash->{$volid}->{shared} //= 0;
3226         $volhash->{$volid}->{shared} = 1 if $shared;
3227
3228         $volhash->{$volid}->{referenced_in_config} //= 0;
3229         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3230
3231         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3232             if defined($snapname);
3233     };
3234
3235     foreach_drive($conf, sub {
3236         my ($ds, $drive) = @_;
3237         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3238     });
3239
3240     foreach my $snapname (keys %{$conf->{snapshots}}) {
3241         my $snap = $conf->{snapshots}->{$snapname};
3242         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3243         foreach_drive($snap, sub {
3244             my ($ds, $drive) = @_;
3245             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3246         });
3247     }
3248
3249     foreach my $volid (keys %$volhash) {
3250         &$func($volid, $volhash->{$volid}, @param);
3251     }
3252 }
3253
3254 sub conf_has_serial {
3255     my ($conf) = @_;
3256
3257     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3258         if ($conf->{"serial$i"}) {
3259             return 1;
3260         }
3261     }
3262
3263     return 0;
3264 }
3265
3266 sub vga_conf_has_spice {
3267     my ($vga) = @_;
3268
3269     my $vgaconf = parse_vga($vga);
3270     my $vgatype = $vgaconf->{type};
3271     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3272
3273     return $1 || 1;
3274 }
3275
3276 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3277 sub get_host_arch() {
3278     $host_arch = (POSIX::uname())[4] if !$host_arch;
3279     return $host_arch;
3280 }
3281
3282 sub is_native($) {
3283     my ($arch) = @_;
3284     return get_host_arch() eq $arch;
3285 }
3286
3287 my $default_machines = {
3288     x86_64 => 'pc',
3289     aarch64 => 'virt',
3290 };
3291
3292 sub get_basic_machine_info {
3293     my ($conf, $forcemachine) = @_;
3294
3295     my $arch = $conf->{arch} // get_host_arch();
3296     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3297     return ($arch, $machine);
3298 }
3299
3300 sub get_ovmf_files($) {
3301     my ($arch) = @_;
3302
3303     my $ovmf = $OVMF->{$arch}
3304         or die "no OVMF images known for architecture '$arch'\n";
3305
3306     return @$ovmf;
3307 }
3308
3309 my $Arch2Qemu = {
3310     aarch64 => '/usr/bin/qemu-system-aarch64',
3311     x86_64 => '/usr/bin/qemu-system-x86_64',
3312 };
3313 sub get_command_for_arch($) {
3314     my ($arch) = @_;
3315     return '/usr/bin/kvm' if is_native($arch);
3316
3317     my $cmd = $Arch2Qemu->{$arch}
3318         or die "don't know how to emulate architecture '$arch'\n";
3319     return $cmd;
3320 }
3321
3322 sub get_cpu_options {
3323     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3324
3325     my $cpuFlags = [];
3326     my $ostype = $conf->{ostype};
3327
3328     my $cpu = $kvm ? "kvm64" : "qemu64";
3329     if ($arch eq 'aarch64') {
3330         $cpu = 'cortex-a57';
3331     }
3332     if (my $cputype = $conf->{cpu}) {
3333         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3334             or die "Cannot parse cpu description: $cputype\n";
3335         $cpu = $cpuconf->{cputype};
3336         $kvm_off = 1 if $cpuconf->{hidden};
3337
3338         if (defined(my $flags = $cpuconf->{flags})) {
3339             push @$cpuFlags, split(";", $flags);
3340         }
3341     }
3342
3343     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3344
3345     push @$cpuFlags , '-x2apic'
3346         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3347
3348     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3349
3350     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3351
3352     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3353
3354         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3355         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3356     }
3357
3358     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3359
3360     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3361
3362     push @$cpuFlags, 'kvm=off' if $kvm_off;
3363
3364     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3365         push @$cpuFlags, "vendor=${cpu_vendor}"
3366             if $cpu_vendor ne 'default';
3367     } elsif ($arch ne 'aarch64') {
3368         die "internal error"; # should not happen
3369     }
3370
3371     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3372
3373     return ('-cpu', $cpu);
3374 }
3375
3376 sub config_to_command {
3377     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3378
3379     my $cmd = [];
3380     my $globalFlags = [];
3381     my $machineFlags = [];
3382     my $rtcFlags = [];
3383     my $devices = [];
3384     my $pciaddr = '';
3385     my $bridges = {};
3386     my $kvmver = kvm_user_version();
3387     my $vernum = 0; # unknown
3388     my $ostype = $conf->{ostype};
3389     my $winversion = windows_version($ostype);
3390     my $kvm = $conf->{kvm};
3391
3392     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3393     $kvm //= 1 if is_native($arch);
3394
3395     if ($kvm) {
3396         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3397             if !defined kvm_version();
3398     }
3399
3400     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3401         $vernum = $1*1000000+$2*1000;
3402     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3403         $vernum = $1*1000000+$2*1000+$3;
3404     }
3405
3406     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3407
3408     my $have_ovz = -f '/proc/vz/vestat';
3409
3410     my $q35 = machine_type_is_q35($conf);
3411     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3412     my $use_old_bios_files = undef;
3413     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3414
3415     my $cpuunits = defined($conf->{cpuunits}) ?
3416             $conf->{cpuunits} : $defaults->{cpuunits};
3417
3418     push @$cmd, get_command_for_arch($arch);
3419
3420     push @$cmd, '-id', $vmid;
3421
3422     my $vmname = $conf->{name} || "vm$vmid";
3423
3424     push @$cmd, '-name', $vmname;
3425
3426     my $use_virtio = 0;
3427
3428     my $qmpsocket = qmp_socket($vmid);
3429     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3430     push @$cmd, '-mon', "chardev=qmp,mode=control";
3431
3432     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3433         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3434         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3435     }
3436
3437     push @$cmd, '-pidfile' , pidfile_name($vmid);
3438
3439     push @$cmd, '-daemonize';
3440
3441     if ($conf->{smbios1}) {
3442         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3443     }
3444
3445     if ($conf->{vmgenid}) {
3446         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3447     }
3448
3449     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3450     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3451         die "uefi base image not found\n" if ! -f $ovmf_code;
3452
3453         my $path;
3454         my $format;
3455         if (my $efidisk = $conf->{efidisk0}) {
3456             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3457             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3458             $format = $d->{format};
3459             if ($storeid) {
3460                 $path = PVE::Storage::path($storecfg, $d->{file});
3461                 if (!defined($format)) {
3462                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3463                     $format = qemu_img_format($scfg, $volname);
3464                 }
3465             } else {
3466                 $path = $d->{file};
3467                 die "efidisk format must be specified\n"
3468                     if !defined($format);
3469             }
3470         } else {
3471             warn "no efidisk configured! Using temporary efivars disk.\n";
3472             $path = "/tmp/$vmid-ovmf.fd";
3473             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3474             $format = 'raw';
3475         }
3476
3477         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3478         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3479     }
3480
3481
3482     # add usb controllers
3483     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3484     push @$devices, @usbcontrollers if @usbcontrollers;
3485     my $vga = parse_vga($conf->{vga});
3486
3487     my $qxlnum = vga_conf_has_spice($conf->{vga});
3488     $vga->{type} = 'qxl' if $qxlnum;
3489
3490     if (!$vga->{type}) {
3491         if ($arch eq 'aarch64') {
3492             $vga->{type} = 'virtio';
3493         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3494             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3495         } else {
3496             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3497         }
3498     }
3499
3500     # enable absolute mouse coordinates (needed by vnc)
3501     my $tablet;
3502     if (defined($conf->{tablet})) {
3503         $tablet = $conf->{tablet};
3504     } else {
3505         $tablet = $defaults->{tablet};
3506         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3507         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3508     }
3509
3510     if ($tablet) {
3511         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3512         my $kbd = print_keyboarddevice_full($conf, $arch);
3513         push @$devices, '-device', $kbd if defined($kbd);
3514     }
3515
3516     my $kvm_off = 0;
3517     my $gpu_passthrough;
3518
3519     # host pci devices
3520     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3521         my $d = parse_hostpci($conf->{"hostpci$i"});
3522         next if !$d;
3523
3524         my $pcie = $d->{pcie};
3525         if($pcie){
3526             die "q35 machine model is not enabled" if !$q35;
3527             # win7 wants to have the pcie devices directly on the pcie bus
3528             # instead of in the root port
3529             if ($winversion == 7) {
3530                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3531             } else {
3532                 $pciaddr = print_pcie_addr("hostpci$i");
3533             }
3534         }else{
3535             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3536         }
3537
3538         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3539         my $romfile = $d->{romfile};
3540
3541         my $xvga = '';
3542         if ($d->{'x-vga'}) {
3543             $xvga = ',x-vga=on';
3544             $kvm_off = 1;
3545             $vga->{type} = 'none';
3546             $gpu_passthrough = 1;
3547
3548             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3549                 $xvga = "";
3550             }
3551         }
3552         my $pcidevices = $d->{pciid};
3553         my $multifunction = 1 if @$pcidevices > 1;
3554         my $sysfspath;
3555         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3556             my $id = $pcidevices->[0]->{id};
3557             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3558             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3559         } elsif ($d->{mdev}) {
3560             warn "ignoring mediated device with multifunction device\n";
3561         }
3562
3563         my $j=0;
3564         foreach my $pcidevice (@$pcidevices) {
3565
3566             my $id = "hostpci$i";
3567             $id .= ".$j" if $multifunction;
3568             my $addr = $pciaddr;
3569             $addr .= ".$j" if $multifunction;
3570             my $devicestr = "vfio-pci";
3571             if ($sysfspath) {
3572                 $devicestr .= ",sysfsdev=$sysfspath";
3573             } else {
3574                 $devicestr .= ",host=$pcidevice->{id}";
3575             }
3576             $devicestr .= ",id=$id$addr";
3577
3578             if($j == 0){
3579                 $devicestr .= "$rombar$xvga";
3580                 $devicestr .= ",multifunction=on" if $multifunction;
3581                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3582             }
3583
3584             push @$devices, '-device', $devicestr;
3585             $j++;
3586         }
3587     }
3588
3589     # usb devices
3590     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3591     push @$devices, @usbdevices if @usbdevices;
3592     # serial devices
3593     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3594         if (my $path = $conf->{"serial$i"}) {
3595             if ($path eq 'socket') {
3596                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3597                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3598                 # On aarch64, serial0 is the UART device. Qemu only allows
3599                 # connecting UART devices via the '-serial' command line, as
3600                 # the device has a fixed slot on the hardware...
3601                 if ($arch eq 'aarch64' && $i == 0) {
3602                     push @$devices, '-serial', "chardev:serial$i";
3603                 } else {
3604                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3605                 }
3606             } else {
3607                 die "no such serial device\n" if ! -c $path;
3608                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3609                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3610             }
3611         }
3612     }
3613
3614     # parallel devices
3615     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3616         if (my $path = $conf->{"parallel$i"}) {
3617             die "no such parallel device\n" if ! -c $path;
3618             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3619             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3620             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3621         }
3622     }
3623
3624
3625     my $sockets = 1;
3626     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3627     $sockets = $conf->{sockets} if  $conf->{sockets};
3628
3629     my $cores = $conf->{cores} || 1;
3630
3631     my $maxcpus = $sockets * $cores;
3632
3633     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3634
3635     my $allowed_vcpus = $cpuinfo->{cpus};
3636
3637     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3638         if ($allowed_vcpus < $maxcpus);
3639
3640     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3641
3642         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3643         for (my $i = 2; $i <= $vcpus; $i++)  {
3644             my $cpustr = print_cpu_device($conf,$i);
3645             push @$cmd, '-device', $cpustr;
3646         }
3647
3648     } else {
3649
3650         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3651     }
3652     push @$cmd, '-nodefaults';
3653
3654     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3655
3656     my $bootindex_hash = {};
3657     my $i = 1;
3658     foreach my $o (split(//, $bootorder)) {
3659         $bootindex_hash->{$o} = $i*100;
3660         $i++;
3661     }
3662
3663     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3664
3665     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3666
3667     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3668
3669     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3670         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3671         my $socket = vnc_socket($vmid);
3672         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3673     } else {
3674         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3675         push @$cmd, '-nographic';
3676     }
3677
3678     # time drift fix
3679     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3680
3681     my $useLocaltime = $conf->{localtime};
3682
3683     if ($winversion >= 5) { # windows
3684         $useLocaltime = 1 if !defined($conf->{localtime});
3685
3686         # use time drift fix when acpi is enabled
3687         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3688             $tdf = 1 if !defined($conf->{tdf});
3689         }
3690     }
3691
3692     if ($winversion >= 6) {
3693         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3694         push @$cmd, '-no-hpet';
3695     }
3696
3697     push @$rtcFlags, 'driftfix=slew' if $tdf;
3698
3699     if (!$kvm) {
3700         push @$machineFlags, 'accel=tcg';
3701     }
3702
3703     if ($machine_type) {
3704         push @$machineFlags, "type=${machine_type}";
3705     }
3706
3707     if ($conf->{startdate}) {
3708         push @$rtcFlags, "base=$conf->{startdate}";
3709     } elsif ($useLocaltime) {
3710         push @$rtcFlags, 'base=localtime';
3711     }
3712
3713     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3714
3715     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3716
3717     push @$cmd, '-S' if $conf->{freeze};
3718
3719     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3720
3721     # enable sound
3722     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3723     #push @$cmd, '-soundhw', 'es1370';
3724     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3725
3726     if (parse_guest_agent($conf)->{enabled}) {
3727         my $qgasocket = qmp_socket($vmid, 1);
3728         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3729         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3730         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3731         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3732     }
3733
3734     my $spice_port;
3735
3736     if ($qxlnum) {
3737         if ($qxlnum > 1) {
3738             if ($winversion){
3739                 for(my $i = 1; $i < $qxlnum; $i++){
3740                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3741                 }
3742             } else {
3743                 # assume other OS works like Linux
3744                 my ($ram, $vram) = ("134217728", "67108864");
3745                 if ($vga->{memory}) {
3746                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3747                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3748                 }
3749                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3750                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3751             }
3752         }
3753
3754         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3755
3756         my $nodename = PVE::INotify::nodename();
3757         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3758         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3759         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3760         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3761         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3762
3763         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3764
3765         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3766         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3767         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3768     }
3769
3770     # enable balloon by default, unless explicitly disabled
3771     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3772         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3773         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3774     }
3775
3776     if ($conf->{watchdog}) {
3777         my $wdopts = parse_watchdog($conf->{watchdog});
3778         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3779         my $watchdog = $wdopts->{model} || 'i6300esb';
3780         push @$devices, '-device', "$watchdog$pciaddr";
3781         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3782     }
3783
3784     my $vollist = [];
3785     my $scsicontroller = {};
3786     my $ahcicontroller = {};
3787     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3788
3789     # Add iscsi initiator name if available
3790     if (my $initiator = get_initiator_name()) {
3791         push @$devices, '-iscsi', "initiator-name=$initiator";
3792     }
3793
3794     foreach_drive($conf, sub {
3795         my ($ds, $drive) = @_;
3796
3797         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3798             push @$vollist, $drive->{file};
3799         }
3800
3801         # ignore efidisk here, already added in bios/fw handling code above
3802         return if $drive->{interface} eq 'efidisk';
3803
3804         $use_virtio = 1 if $ds =~ m/^virtio/;
3805
3806         if (drive_is_cdrom ($drive)) {
3807             if ($bootindex_hash->{d}) {
3808                 $drive->{bootindex} = $bootindex_hash->{d};
3809                 $bootindex_hash->{d} += 1;
3810             }
3811         } else {
3812             if ($bootindex_hash->{c}) {
3813                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3814                 $bootindex_hash->{c} += 1;
3815             }
3816         }
3817
3818         if($drive->{interface} eq 'virtio'){
3819            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3820         }
3821
3822         if ($drive->{interface} eq 'scsi') {
3823
3824             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3825
3826             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3827             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3828
3829             my $iothread = '';
3830             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3831                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3832                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3833             } elsif ($drive->{iothread}) {
3834                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3835             }
3836
3837             my $queues = '';
3838             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3839                 $queues = ",num_queues=$drive->{queues}";
3840             }
3841
3842             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3843             $scsicontroller->{$controller}=1;
3844         }
3845
3846         if ($drive->{interface} eq 'sata') {
3847            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3848            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3849            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3850            $ahcicontroller->{$controller}=1;
3851         }
3852
3853         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3854         push @$devices, '-drive',$drive_cmd;
3855         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3856     });
3857
3858     for (my $i = 0; $i < $MAX_NETS; $i++) {
3859          next if !$conf->{"net$i"};
3860          my $d = parse_net($conf->{"net$i"});
3861          next if !$d;
3862
3863          $use_virtio = 1 if $d->{model} eq 'virtio';
3864
3865          if ($bootindex_hash->{n}) {
3866             $d->{bootindex} = $bootindex_hash->{n};
3867             $bootindex_hash->{n} += 1;
3868          }
3869
3870          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3871          push @$devices, '-netdev', $netdevfull;
3872
3873          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3874          push @$devices, '-device', $netdevicefull;
3875     }
3876
3877     if (!$q35) {
3878         # add pci bridges
3879         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3880            $bridges->{1} = 1;
3881            $bridges->{2} = 1;
3882         }
3883
3884         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3885
3886         while (my ($k, $v) = each %$bridges) {
3887             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3888             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3889         }
3890     }
3891
3892     push @$cmd, @$devices;
3893     push @$cmd, '-rtc', join(',', @$rtcFlags)
3894         if scalar(@$rtcFlags);
3895     push @$cmd, '-machine', join(',', @$machineFlags)
3896         if scalar(@$machineFlags);
3897     push @$cmd, '-global', join(',', @$globalFlags)
3898         if scalar(@$globalFlags);
3899
3900     # add custom args
3901     if ($conf->{args}) {
3902         my $aa = PVE::Tools::split_args($conf->{args});
3903         push @$cmd, @$aa;
3904     }
3905
3906     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3907 }
3908
3909 sub vnc_socket {
3910     my ($vmid) = @_;
3911     return "${var_run_tmpdir}/$vmid.vnc";
3912 }
3913
3914 sub spice_port {
3915     my ($vmid) = @_;
3916
3917     my $res = vm_mon_cmd($vmid, 'query-spice');
3918
3919     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3920 }
3921
3922 sub qmp_socket {
3923     my ($vmid, $qga, $name) = @_;
3924     my $sockettype = $qga ? 'qga' : 'qmp';
3925     my $ext = $name ? '-'.$name : '';
3926     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3927 }
3928
3929 sub pidfile_name {
3930     my ($vmid) = @_;
3931     return "${var_run_tmpdir}/$vmid.pid";
3932 }
3933
3934 sub vm_devices_list {
3935     my ($vmid) = @_;
3936
3937     my $res = vm_mon_cmd($vmid, 'query-pci');
3938     my $devices_to_check = [];
3939     my $devices = {};
3940     foreach my $pcibus (@$res) {
3941         push @$devices_to_check, @{$pcibus->{devices}},
3942     }
3943
3944     while (@$devices_to_check) {
3945         my $to_check = [];
3946         for my $d (@$devices_to_check) {
3947             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3948             next if !$d->{'pci_bridge'};
3949
3950             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3951             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3952         }
3953         $devices_to_check = $to_check;
3954     }
3955
3956     my $resblock = vm_mon_cmd($vmid, 'query-block');
3957     foreach my $block (@$resblock) {
3958         if($block->{device} =~ m/^drive-(\S+)/){
3959                 $devices->{$1} = 1;
3960         }
3961     }
3962
3963     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3964     foreach my $mice (@$resmice) {
3965         if ($mice->{name} eq 'QEMU HID Tablet') {
3966             $devices->{tablet} = 1;
3967             last;
3968         }
3969     }
3970
3971     # for usb devices there is no query-usb
3972     # but we can iterate over the entries in
3973     # qom-list path=/machine/peripheral
3974     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3975     foreach my $per (@$resperipheral) {
3976         if ($per->{name} =~ m/^usb\d+$/) {
3977             $devices->{$per->{name}} = 1;
3978         }
3979     }
3980
3981     return $devices;
3982 }
3983
3984 sub vm_deviceplug {
3985     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3986
3987     my $q35 = machine_type_is_q35($conf);
3988
3989     my $devices_list = vm_devices_list($vmid);
3990     return 1 if defined($devices_list->{$deviceid});
3991
3992     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3993
3994     if ($deviceid eq 'tablet') {
3995
3996         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3997
3998     } elsif ($deviceid eq 'keyboard') {
3999
4000         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4001
4002     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4003
4004         die "usb hotplug currently not reliable\n";
4005         # since we can't reliably hot unplug all added usb devices
4006         # and usb passthrough disables live migration
4007         # we disable usb hotplugging for now
4008         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4009
4010     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4011
4012         qemu_iothread_add($vmid, $deviceid, $device);
4013
4014         qemu_driveadd($storecfg, $vmid, $device);
4015         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4016
4017         qemu_deviceadd($vmid, $devicefull);
4018         eval { qemu_deviceaddverify($vmid, $deviceid); };
4019         if (my $err = $@) {
4020             eval { qemu_drivedel($vmid, $deviceid); };
4021             warn $@ if $@;
4022             die $err;
4023         }
4024
4025     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4026
4027
4028         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4029         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4030         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4031
4032         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4033
4034         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4035             qemu_iothread_add($vmid, $deviceid, $device);
4036             $devicefull .= ",iothread=iothread-$deviceid";
4037         }
4038
4039         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4040             $devicefull .= ",num_queues=$device->{queues}";
4041         }
4042
4043         qemu_deviceadd($vmid, $devicefull);
4044         qemu_deviceaddverify($vmid, $deviceid);
4045
4046     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4047
4048         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4049         qemu_driveadd($storecfg, $vmid, $device);
4050
4051         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4052         eval { qemu_deviceadd($vmid, $devicefull); };
4053         if (my $err = $@) {
4054             eval { qemu_drivedel($vmid, $deviceid); };
4055             warn $@ if $@;
4056             die $err;
4057         }
4058
4059     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4060
4061         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4062
4063         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4064         my $use_old_bios_files = undef;
4065         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4066
4067         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4068         qemu_deviceadd($vmid, $netdevicefull);
4069         eval { qemu_deviceaddverify($vmid, $deviceid); };
4070         if (my $err = $@) {
4071             eval { qemu_netdevdel($vmid, $deviceid); };
4072             warn $@ if $@;
4073             die $err;
4074         }
4075
4076     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4077
4078         my $bridgeid = $2;
4079         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4080         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4081
4082         qemu_deviceadd($vmid, $devicefull);
4083         qemu_deviceaddverify($vmid, $deviceid);
4084
4085     } else {
4086         die "can't hotplug device '$deviceid'\n";
4087     }
4088
4089     return 1;
4090 }
4091
4092 # fixme: this should raise exceptions on error!
4093 sub vm_deviceunplug {
4094     my ($vmid, $conf, $deviceid) = @_;
4095
4096     my $devices_list = vm_devices_list($vmid);
4097     return 1 if !defined($devices_list->{$deviceid});
4098
4099     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4100
4101     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4102
4103         qemu_devicedel($vmid, $deviceid);
4104
4105     } elsif ($deviceid =~ m/^usb\d+$/) {
4106
4107         die "usb hotplug currently not reliable\n";
4108         # when unplugging usb devices this way,
4109         # there may be remaining usb controllers/hubs
4110         # so we disable it for now
4111         qemu_devicedel($vmid, $deviceid);
4112         qemu_devicedelverify($vmid, $deviceid);
4113
4114     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4115
4116         qemu_devicedel($vmid, $deviceid);
4117         qemu_devicedelverify($vmid, $deviceid);
4118         qemu_drivedel($vmid, $deviceid);
4119         qemu_iothread_del($conf, $vmid, $deviceid);
4120
4121     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4122
4123         qemu_devicedel($vmid, $deviceid);
4124         qemu_devicedelverify($vmid, $deviceid);
4125         qemu_iothread_del($conf, $vmid, $deviceid);
4126
4127     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4128
4129         qemu_devicedel($vmid, $deviceid);
4130         qemu_drivedel($vmid, $deviceid);
4131         qemu_deletescsihw($conf, $vmid, $deviceid);
4132
4133     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4134
4135         qemu_devicedel($vmid, $deviceid);
4136         qemu_devicedelverify($vmid, $deviceid);
4137         qemu_netdevdel($vmid, $deviceid);
4138
4139     } else {
4140         die "can't unplug device '$deviceid'\n";
4141     }
4142
4143     return 1;
4144 }
4145
4146 sub qemu_deviceadd {
4147     my ($vmid, $devicefull) = @_;
4148
4149     $devicefull = "driver=".$devicefull;
4150     my %options =  split(/[=,]/, $devicefull);
4151
4152     vm_mon_cmd($vmid, "device_add" , %options);
4153 }
4154
4155 sub qemu_devicedel {
4156     my ($vmid, $deviceid) = @_;
4157
4158     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4159 }
4160
4161 sub qemu_iothread_add {
4162     my($vmid, $deviceid, $device) = @_;
4163
4164     if ($device->{iothread}) {
4165         my $iothreads = vm_iothreads_list($vmid);
4166         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4167     }
4168 }
4169
4170 sub qemu_iothread_del {
4171     my($conf, $vmid, $deviceid) = @_;
4172
4173     my $device = parse_drive($deviceid, $conf->{$deviceid});
4174     if ($device->{iothread}) {
4175         my $iothreads = vm_iothreads_list($vmid);
4176         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4177     }
4178 }
4179
4180 sub qemu_objectadd {
4181     my($vmid, $objectid, $qomtype) = @_;
4182
4183     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4184
4185     return 1;
4186 }
4187
4188 sub qemu_objectdel {
4189     my($vmid, $objectid) = @_;
4190
4191     vm_mon_cmd($vmid, "object-del", id => $objectid);
4192
4193     return 1;
4194 }
4195
4196 sub qemu_driveadd {
4197     my ($storecfg, $vmid, $device) = @_;
4198
4199     my $drive = print_drive_full($storecfg, $vmid, $device);
4200     $drive =~ s/\\/\\\\/g;
4201     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4202
4203     # If the command succeeds qemu prints: "OK"
4204     return 1 if $ret =~ m/OK/s;
4205
4206     die "adding drive failed: $ret\n";
4207 }
4208
4209 sub qemu_drivedel {
4210     my($vmid, $deviceid) = @_;
4211
4212     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4213     $ret =~ s/^\s+//;
4214
4215     return 1 if $ret eq "";
4216
4217     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4218     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4219
4220     die "deleting drive $deviceid failed : $ret\n";
4221 }
4222
4223 sub qemu_deviceaddverify {
4224     my ($vmid, $deviceid) = @_;
4225
4226     for (my $i = 0; $i <= 5; $i++) {
4227          my $devices_list = vm_devices_list($vmid);
4228          return 1 if defined($devices_list->{$deviceid});
4229          sleep 1;
4230     }
4231
4232     die "error on hotplug device '$deviceid'\n";
4233 }
4234
4235
4236 sub qemu_devicedelverify {
4237     my ($vmid, $deviceid) = @_;
4238
4239     # need to verify that the device is correctly removed as device_del
4240     # is async and empty return is not reliable
4241
4242     for (my $i = 0; $i <= 5; $i++) {
4243          my $devices_list = vm_devices_list($vmid);
4244          return 1 if !defined($devices_list->{$deviceid});
4245          sleep 1;
4246     }
4247
4248     die "error on hot-unplugging device '$deviceid'\n";
4249 }
4250
4251 sub qemu_findorcreatescsihw {
4252     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4253
4254     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4255
4256     my $scsihwid="$controller_prefix$controller";
4257     my $devices_list = vm_devices_list($vmid);
4258
4259     if(!defined($devices_list->{$scsihwid})) {
4260         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4261     }
4262
4263     return 1;
4264 }
4265
4266 sub qemu_deletescsihw {
4267     my ($conf, $vmid, $opt) = @_;
4268
4269     my $device = parse_drive($opt, $conf->{$opt});
4270
4271     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4272         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4273         return 1;
4274     }
4275
4276     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4277
4278     my $devices_list = vm_devices_list($vmid);
4279     foreach my $opt (keys %{$devices_list}) {
4280         if (PVE::QemuServer::is_valid_drivename($opt)) {
4281             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4282             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4283                 return 1;
4284             }
4285         }
4286     }
4287
4288     my $scsihwid="scsihw$controller";
4289
4290     vm_deviceunplug($vmid, $conf, $scsihwid);
4291
4292     return 1;
4293 }
4294
4295 sub qemu_add_pci_bridge {
4296     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4297
4298     my $bridges = {};
4299
4300     my $bridgeid;
4301
4302     print_pci_addr($device, $bridges, $arch, $machine_type);
4303
4304     while (my ($k, $v) = each %$bridges) {
4305         $bridgeid = $k;
4306     }
4307     return 1 if !defined($bridgeid) || $bridgeid < 1;
4308
4309     my $bridge = "pci.$bridgeid";
4310     my $devices_list = vm_devices_list($vmid);
4311
4312     if (!defined($devices_list->{$bridge})) {
4313         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4314     }
4315
4316     return 1;
4317 }
4318
4319 sub qemu_set_link_status {
4320     my ($vmid, $device, $up) = @_;
4321
4322     vm_mon_cmd($vmid, "set_link", name => $device,
4323                up => $up ? JSON::true : JSON::false);
4324 }
4325
4326 sub qemu_netdevadd {
4327     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4328
4329     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4330     my %options =  split(/[=,]/, $netdev);
4331
4332     vm_mon_cmd($vmid, "netdev_add",  %options);
4333     return 1;
4334 }
4335
4336 sub qemu_netdevdel {
4337     my ($vmid, $deviceid) = @_;
4338
4339     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4340 }
4341
4342 sub qemu_usb_hotplug {
4343     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4344
4345     return if !$device;
4346
4347     # remove the old one first
4348     vm_deviceunplug($vmid, $conf, $deviceid);
4349
4350     # check if xhci controller is necessary and available
4351     if ($device->{usb3}) {
4352
4353         my $devicelist = vm_devices_list($vmid);
4354
4355         if (!$devicelist->{xhci}) {
4356             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4357             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4358         }
4359     }
4360     my $d = parse_usb_device($device->{host});
4361     $d->{usb3} = $device->{usb3};
4362
4363     # add the new one
4364     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4365 }
4366
4367 sub qemu_cpu_hotplug {
4368     my ($vmid, $conf, $vcpus) = @_;
4369
4370     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4371
4372     my $sockets = 1;
4373     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4374     $sockets = $conf->{sockets} if  $conf->{sockets};
4375     my $cores = $conf->{cores} || 1;
4376     my $maxcpus = $sockets * $cores;
4377
4378     $vcpus = $maxcpus if !$vcpus;
4379
4380     die "you can't add more vcpus than maxcpus\n"
4381         if $vcpus > $maxcpus;
4382
4383     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4384
4385     if ($vcpus < $currentvcpus) {
4386
4387         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4388
4389             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4390                 qemu_devicedel($vmid, "cpu$i");
4391                 my $retry = 0;
4392                 my $currentrunningvcpus = undef;
4393                 while (1) {
4394                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4395                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4396                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4397                     $retry++;
4398                     sleep 1;
4399                 }
4400                 #update conf after each succesfull cpu unplug
4401                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4402                 PVE::QemuConfig->write_config($vmid, $conf);
4403             }
4404         } else {
4405             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4406         }
4407
4408         return;
4409     }
4410
4411     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4412     die "vcpus in running vm does not match its configuration\n"
4413         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4414
4415     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4416
4417         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4418             my $cpustr = print_cpu_device($conf, $i);
4419             qemu_deviceadd($vmid, $cpustr);
4420
4421             my $retry = 0;
4422             my $currentrunningvcpus = undef;
4423             while (1) {
4424                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4425                 last if scalar(@{$currentrunningvcpus}) == $i;
4426                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4427                 sleep 1;
4428                 $retry++;
4429             }
4430             #update conf after each succesfull cpu hotplug
4431             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4432             PVE::QemuConfig->write_config($vmid, $conf);
4433         }
4434     } else {
4435
4436         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4437             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4438         }
4439     }
4440 }
4441
4442 sub qemu_block_set_io_throttle {
4443     my ($vmid, $deviceid,
4444         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4445         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4446         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4447         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4448
4449     return if !check_running($vmid) ;
4450
4451     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4452         bps => int($bps),
4453         bps_rd => int($bps_rd),
4454         bps_wr => int($bps_wr),
4455         iops => int($iops),
4456         iops_rd => int($iops_rd),
4457         iops_wr => int($iops_wr),
4458         bps_max => int($bps_max),
4459         bps_rd_max => int($bps_rd_max),
4460         bps_wr_max => int($bps_wr_max),
4461         iops_max => int($iops_max),
4462         iops_rd_max => int($iops_rd_max),
4463         iops_wr_max => int($iops_wr_max),
4464         bps_max_length => int($bps_max_length),
4465         bps_rd_max_length => int($bps_rd_max_length),
4466         bps_wr_max_length => int($bps_wr_max_length),
4467         iops_max_length => int($iops_max_length),
4468         iops_rd_max_length => int($iops_rd_max_length),
4469         iops_wr_max_length => int($iops_wr_max_length),
4470     );
4471
4472 }
4473
4474 # old code, only used to shutdown old VM after update
4475 sub __read_avail {
4476     my ($fh, $timeout) = @_;
4477
4478     my $sel = new IO::Select;
4479     $sel->add($fh);
4480
4481     my $res = '';
4482     my $buf;
4483
4484     my @ready;
4485     while (scalar (@ready = $sel->can_read($timeout))) {
4486         my $count;
4487         if ($count = $fh->sysread($buf, 8192)) {
4488             if ($buf =~ /^(.*)\(qemu\) $/s) {
4489                 $res .= $1;
4490                 last;
4491             } else {
4492                 $res .= $buf;
4493             }
4494         } else {
4495             if (!defined($count)) {
4496                 die "$!\n";
4497             }
4498             last;
4499         }
4500     }
4501
4502     die "monitor read timeout\n" if !scalar(@ready);
4503
4504     return $res;
4505 }
4506
4507 sub qemu_block_resize {
4508     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4509
4510     my $running = check_running($vmid);
4511
4512     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4513
4514     return if !$running;
4515
4516     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4517
4518 }
4519
4520 sub qemu_volume_snapshot {
4521     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4522
4523     my $running = check_running($vmid);
4524
4525     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4526         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4527     } else {
4528         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4529     }
4530 }
4531
4532 sub qemu_volume_snapshot_delete {
4533     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4534
4535     my $running = check_running($vmid);
4536
4537     if($running) {
4538
4539         $running = undef;
4540         my $conf = PVE::QemuConfig->load_config($vmid);
4541         foreach_drive($conf, sub {
4542             my ($ds, $drive) = @_;
4543             $running = 1 if $drive->{file} eq $volid;
4544         });
4545     }
4546
4547     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4548         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4549     } else {
4550         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4551     }
4552 }
4553
4554 sub set_migration_caps {
4555     my ($vmid) = @_;
4556
4557     my $cap_ref = [];
4558
4559     my $enabled_cap = {
4560         "auto-converge" => 1,
4561         "xbzrle" => 1,
4562         "x-rdma-pin-all" => 0,
4563         "zero-blocks" => 0,
4564         "compress" => 0
4565     };
4566
4567     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4568
4569     for my $supported_capability (@$supported_capabilities) {
4570         push @$cap_ref, {
4571             capability => $supported_capability->{capability},
4572             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4573         };
4574     }
4575
4576     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4577 }
4578
4579 my $fast_plug_option = {
4580     'lock' => 1,
4581     'name' => 1,
4582     'onboot' => 1,
4583     'shares' => 1,
4584     'startup' => 1,
4585     'description' => 1,
4586     'protection' => 1,
4587     'vmstatestorage' => 1,
4588 };
4589
4590 # hotplug changes in [PENDING]
4591 # $selection hash can be used to only apply specified options, for
4592 # example: { cores => 1 } (only apply changed 'cores')
4593 # $errors ref is used to return error messages
4594 sub vmconfig_hotplug_pending {
4595     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4596
4597     my $defaults = load_defaults();
4598     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4599
4600     # commit values which do not have any impact on running VM first
4601     # Note: those option cannot raise errors, we we do not care about
4602     # $selection and always apply them.
4603
4604     my $add_error = sub {
4605         my ($opt, $msg) = @_;
4606         $errors->{$opt} = "hotplug problem - $msg";
4607     };
4608
4609     my $changes = 0;
4610     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4611         if ($fast_plug_option->{$opt}) {
4612             $conf->{$opt} = $conf->{pending}->{$opt};
4613             delete $conf->{pending}->{$opt};
4614             $changes = 1;
4615         }
4616     }
4617
4618     if ($changes) {
4619         PVE::QemuConfig->write_config($vmid, $conf);
4620         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4621     }
4622
4623     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4624
4625     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4626     while (my ($opt, $force) = each %$pending_delete_hash) {
4627         next if $selection && !$selection->{$opt};
4628         eval {
4629             if ($opt eq 'hotplug') {
4630                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4631             } elsif ($opt eq 'tablet') {
4632                 die "skip\n" if !$hotplug_features->{usb};
4633                 if ($defaults->{tablet}) {
4634                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4635                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4636                         if $arch eq 'aarch64';
4637                 } else {
4638                     vm_deviceunplug($vmid, $conf, 'tablet');
4639                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4640                 }
4641             } elsif ($opt =~ m/^usb\d+/) {
4642                 die "skip\n";
4643                 # since we cannot reliably hot unplug usb devices
4644                 # we are disabling it
4645                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4646                 vm_deviceunplug($vmid, $conf, $opt);
4647             } elsif ($opt eq 'vcpus') {
4648                 die "skip\n" if !$hotplug_features->{cpu};
4649                 qemu_cpu_hotplug($vmid, $conf, undef);
4650             } elsif ($opt eq 'balloon') {
4651                 # enable balloon device is not hotpluggable
4652                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4653                 # here we reset the ballooning value to memory
4654                 my $balloon = $conf->{memory} || $defaults->{memory};
4655                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4656             } elsif ($fast_plug_option->{$opt}) {
4657                 # do nothing
4658             } elsif ($opt =~ m/^net(\d+)$/) {
4659                 die "skip\n" if !$hotplug_features->{network};
4660                 vm_deviceunplug($vmid, $conf, $opt);
4661             } elsif (is_valid_drivename($opt)) {
4662                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4663                 vm_deviceunplug($vmid, $conf, $opt);
4664                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4665             } elsif ($opt =~ m/^memory$/) {
4666                 die "skip\n" if !$hotplug_features->{memory};
4667                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4668             } elsif ($opt eq 'cpuunits') {
4669                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4670             } elsif ($opt eq 'cpulimit') {
4671                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4672             } else {
4673                 die "skip\n";
4674             }
4675         };
4676         if (my $err = $@) {
4677             &$add_error($opt, $err) if $err ne "skip\n";
4678         } else {
4679             # save new config if hotplug was successful
4680             delete $conf->{$opt};
4681             vmconfig_undelete_pending_option($conf, $opt);
4682             PVE::QemuConfig->write_config($vmid, $conf);
4683             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4684         }
4685     }
4686
4687     my $apply_pending_cloudinit;
4688     $apply_pending_cloudinit = sub {
4689         my ($key, $value) = @_;
4690         $apply_pending_cloudinit = sub {}; # once is enough
4691
4692         my @cloudinit_opts = keys %$confdesc_cloudinit;
4693         foreach my $opt (keys %{$conf->{pending}}) {
4694             next if !grep { $_ eq $opt } @cloudinit_opts;
4695             $conf->{$opt} = delete $conf->{pending}->{$opt};
4696         }
4697
4698         my $new_conf = { %$conf };
4699         $new_conf->{$key} = $value;
4700         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4701     };
4702
4703     foreach my $opt (keys %{$conf->{pending}}) {
4704         next if $selection && !$selection->{$opt};
4705         my $value = $conf->{pending}->{$opt};
4706         eval {
4707             if ($opt eq 'hotplug') {
4708                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4709             } elsif ($opt eq 'tablet') {
4710                 die "skip\n" if !$hotplug_features->{usb};
4711                 if ($value == 1) {
4712                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4713                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4714                         if $arch eq 'aarch64';
4715                 } elsif ($value == 0) {
4716                     vm_deviceunplug($vmid, $conf, 'tablet');
4717                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4718                 }
4719             } elsif ($opt =~ m/^usb\d+$/) {
4720                 die "skip\n";
4721                 # since we cannot reliably hot unplug usb devices
4722                 # we are disabling it
4723                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4724                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4725                 die "skip\n" if !$d;
4726                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4727             } elsif ($opt eq 'vcpus') {
4728                 die "skip\n" if !$hotplug_features->{cpu};
4729                 qemu_cpu_hotplug($vmid, $conf, $value);
4730             } elsif ($opt eq 'balloon') {
4731                 # enable/disable balloning device is not hotpluggable
4732                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4733                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4734                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4735
4736                 # allow manual ballooning if shares is set to zero
4737                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4738                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4739                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4740                 }
4741             } elsif ($opt =~ m/^net(\d+)$/) {
4742                 # some changes can be done without hotplug
4743                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4744                                     $vmid, $opt, $value, $arch, $machine_type);
4745             } elsif (is_valid_drivename($opt)) {
4746                 # some changes can be done without hotplug
4747                 my $drive = parse_drive($opt, $value);
4748                 if (drive_is_cloudinit($drive)) {
4749                     &$apply_pending_cloudinit($opt, $value);
4750                 }
4751                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4752                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4753             } elsif ($opt =~ m/^memory$/) { #dimms
4754                 die "skip\n" if !$hotplug_features->{memory};
4755                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4756             } elsif ($opt eq 'cpuunits') {
4757                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4758             } elsif ($opt eq 'cpulimit') {
4759                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4760                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4761             } else {
4762                 die "skip\n";  # skip non-hot-pluggable options
4763             }
4764         };
4765         if (my $err = $@) {
4766             &$add_error($opt, $err) if $err ne "skip\n";
4767         } else {
4768             # save new config if hotplug was successful
4769             $conf->{$opt} = $value;
4770             delete $conf->{pending}->{$opt};
4771             PVE::QemuConfig->write_config($vmid, $conf);
4772             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4773         }
4774     }
4775 }
4776
4777 sub try_deallocate_drive {
4778     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4779
4780     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4781         my $volid = $drive->{file};
4782         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4783             my $sid = PVE::Storage::parse_volume_id($volid);
4784             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4785
4786             # check if the disk is really unused
4787             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4788                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4789             PVE::Storage::vdisk_free($storecfg, $volid);
4790             return 1;
4791         } else {
4792             # If vm is not owner of this disk remove from config
4793             return 1;
4794         }
4795     }
4796
4797     return undef;
4798 }
4799
4800 sub vmconfig_delete_or_detach_drive {
4801     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4802
4803     my $drive = parse_drive($opt, $conf->{$opt});
4804
4805     my $rpcenv = PVE::RPCEnvironment::get();
4806     my $authuser = $rpcenv->get_user();
4807
4808     if ($force) {
4809         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4810         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4811     } else {
4812         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4813     }
4814 }
4815
4816 sub vmconfig_apply_pending {
4817     my ($vmid, $conf, $storecfg) = @_;
4818
4819     # cold plug
4820
4821     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4822     while (my ($opt, $force) = each %$pending_delete_hash) {
4823         die "internal error" if $opt =~ m/^unused/;
4824         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4825         if (!defined($conf->{$opt})) {
4826             vmconfig_undelete_pending_option($conf, $opt);
4827             PVE::QemuConfig->write_config($vmid, $conf);
4828         } elsif (is_valid_drivename($opt)) {
4829             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4830             vmconfig_undelete_pending_option($conf, $opt);
4831             delete $conf->{$opt};
4832             PVE::QemuConfig->write_config($vmid, $conf);
4833         } else {
4834             vmconfig_undelete_pending_option($conf, $opt);
4835             delete $conf->{$opt};
4836             PVE::QemuConfig->write_config($vmid, $conf);
4837         }
4838     }
4839
4840     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4841
4842     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4843         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4844
4845         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4846             # skip if nothing changed
4847         } elsif (is_valid_drivename($opt)) {
4848             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4849                 if defined($conf->{$opt});
4850             $conf->{$opt} = $conf->{pending}->{$opt};
4851         } else {
4852             $conf->{$opt} = $conf->{pending}->{$opt};
4853         }
4854
4855         delete $conf->{pending}->{$opt};
4856         PVE::QemuConfig->write_config($vmid, $conf);
4857     }
4858 }
4859
4860 my $safe_num_ne = sub {
4861     my ($a, $b) = @_;
4862
4863     return 0 if !defined($a) && !defined($b);
4864     return 1 if !defined($a);
4865     return 1 if !defined($b);
4866
4867     return $a != $b;
4868 };
4869
4870 my $safe_string_ne = sub {
4871     my ($a, $b) = @_;
4872
4873     return 0 if !defined($a) && !defined($b);
4874     return 1 if !defined($a);
4875     return 1 if !defined($b);
4876
4877     return $a ne $b;
4878 };
4879
4880 sub vmconfig_update_net {
4881     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4882
4883     my $newnet = parse_net($value);
4884
4885     if ($conf->{$opt}) {
4886         my $oldnet = parse_net($conf->{$opt});
4887
4888         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4889             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4890             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4891             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4892
4893             # for non online change, we try to hot-unplug
4894             die "skip\n" if !$hotplug;
4895             vm_deviceunplug($vmid, $conf, $opt);
4896         } else {
4897
4898             die "internal error" if $opt !~ m/net(\d+)/;
4899             my $iface = "tap${vmid}i$1";
4900
4901             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4902                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4903                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4904                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4905                 PVE::Network::tap_unplug($iface);
4906                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4907             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4908                 # Rate can be applied on its own but any change above needs to
4909                 # include the rate in tap_plug since OVS resets everything.
4910                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4911             }
4912
4913             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4914                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4915             }
4916
4917             return 1;
4918         }
4919     }
4920
4921     if ($hotplug) {
4922         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4923     } else {
4924         die "skip\n";
4925     }
4926 }
4927
4928 sub vmconfig_update_disk {
4929     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4930
4931     # fixme: do we need force?
4932
4933     my $drive = parse_drive($opt, $value);
4934
4935     if ($conf->{$opt}) {
4936
4937         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4938
4939             my $media = $drive->{media} || 'disk';
4940             my $oldmedia = $old_drive->{media} || 'disk';
4941             die "unable to change media type\n" if $media ne $oldmedia;
4942
4943             if (!drive_is_cdrom($old_drive)) {
4944
4945                 if ($drive->{file} ne $old_drive->{file}) {
4946
4947                     die "skip\n" if !$hotplug;
4948
4949                     # unplug and register as unused
4950                     vm_deviceunplug($vmid, $conf, $opt);
4951                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4952
4953                 } else {
4954                     # update existing disk
4955
4956                     # skip non hotpluggable value
4957                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4958                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4959                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4960                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4961                         die "skip\n";
4962                     }
4963
4964                     # apply throttle
4965                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4966                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4967                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4968                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4969                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4970                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4971                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4972                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4973                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4974                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4975                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4976                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4977                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4978                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4979                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4980                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4981                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4982                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4983
4984                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4985                                                    ($drive->{mbps} || 0)*1024*1024,
4986                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4987                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4988                                                    $drive->{iops} || 0,
4989                                                    $drive->{iops_rd} || 0,
4990                                                    $drive->{iops_wr} || 0,
4991                                                    ($drive->{mbps_max} || 0)*1024*1024,
4992                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4993                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4994                                                    $drive->{iops_max} || 0,
4995                                                    $drive->{iops_rd_max} || 0,
4996                                                    $drive->{iops_wr_max} || 0,
4997                                                    $drive->{bps_max_length} || 1,
4998                                                    $drive->{bps_rd_max_length} || 1,
4999                                                    $drive->{bps_wr_max_length} || 1,
5000                                                    $drive->{iops_max_length} || 1,
5001                                                    $drive->{iops_rd_max_length} || 1,
5002                                                    $drive->{iops_wr_max_length} || 1);
5003
5004                     }
5005
5006                     return 1;
5007                 }
5008
5009             } else { # cdrom
5010
5011                 if ($drive->{file} eq 'none') {
5012                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5013                     if (drive_is_cloudinit($old_drive)) {
5014                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5015                     }
5016                 } else {
5017                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5018                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5019                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5020                 }
5021
5022                 return 1;
5023             }
5024         }
5025     }
5026
5027     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5028     # hotplug new disks
5029     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5030     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5031 }
5032
5033 sub vm_start {
5034     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5035         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5036
5037     PVE::QemuConfig->lock_config($vmid, sub {
5038         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5039
5040         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5041
5042         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5043
5044         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5045
5046         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5047             vmconfig_apply_pending($vmid, $conf, $storecfg);
5048             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5049         }
5050
5051         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5052
5053         my $defaults = load_defaults();
5054
5055         # set environment variable useful inside network script
5056         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5057
5058         my $local_volumes = {};
5059
5060         if ($targetstorage) {
5061             foreach_drive($conf, sub {
5062                 my ($ds, $drive) = @_;
5063
5064                 return if drive_is_cdrom($drive);
5065
5066                 my $volid = $drive->{file};
5067
5068                 return if !$volid;
5069
5070                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5071
5072                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5073                 return if $scfg->{shared};
5074                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5075             });
5076
5077             my $format = undef;
5078
5079             foreach my $opt (sort keys %$local_volumes) {
5080
5081                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5082                 my $drive = parse_drive($opt, $conf->{$opt});
5083
5084                 #if remote storage is specified, use default format
5085                 if ($targetstorage && $targetstorage ne "1") {
5086                     $storeid = $targetstorage;
5087                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5088                     $format = $defFormat;
5089                 } else {
5090                     #else we use same format than original
5091                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5092                     $format = qemu_img_format($scfg, $volid);
5093                 }
5094
5095                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5096                 my $newdrive = $drive;
5097                 $newdrive->{format} = $format;
5098                 $newdrive->{file} = $newvolid;
5099                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5100                 $local_volumes->{$opt} = $drivestr;
5101                 #pass drive to conf for command line
5102                 $conf->{$opt} = $drivestr;
5103             }
5104         }
5105
5106         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5107
5108         my $migrate_port = 0;
5109         my $migrate_uri;
5110         if ($statefile) {
5111             if ($statefile eq 'tcp') {
5112                 my $localip = "localhost";
5113                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5114                 my $nodename = PVE::INotify::nodename();
5115
5116                 if (!defined($migration_type)) {
5117                     if (defined($datacenterconf->{migration}->{type})) {
5118                         $migration_type = $datacenterconf->{migration}->{type};
5119                     } else {
5120                         $migration_type = 'secure';
5121                     }
5122                 }
5123
5124                 if ($migration_type eq 'insecure') {
5125                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5126                     if ($migrate_network_addr) {
5127                         $localip = $migrate_network_addr;
5128                     } else {
5129                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5130                     }
5131
5132                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5133                 }
5134
5135                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5136                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5137                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5138                 push @$cmd, '-incoming', $migrate_uri;
5139                 push @$cmd, '-S';
5140
5141             } elsif ($statefile eq 'unix') {
5142                 # should be default for secure migrations as a ssh TCP forward
5143                 # tunnel is not deterministic reliable ready and fails regurarly
5144                 # to set up in time, so use UNIX socket forwards
5145                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5146                 unlink $socket_addr;
5147
5148                 $migrate_uri = "unix:$socket_addr";
5149
5150                 push @$cmd, '-incoming', $migrate_uri;
5151                 push @$cmd, '-S';
5152
5153             } else {
5154                 push @$cmd, '-loadstate', $statefile;
5155             }
5156         } elsif ($paused) {
5157             push @$cmd, '-S';
5158         }
5159
5160         # host pci devices
5161         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5162           my $d = parse_hostpci($conf->{"hostpci$i"});
5163           next if !$d;
5164           my $pcidevices = $d->{pciid};
5165           foreach my $pcidevice (@$pcidevices) {
5166                 my $pciid = $pcidevice->{id};
5167
5168                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5169                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5170                 die "no pci device info for device '$pciid'\n" if !$info;
5171
5172                 if ($d->{mdev}) {
5173                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5174                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5175                 } else {
5176                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5177                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5178                     die "can't reset pci device '$pciid'\n"
5179                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5180                 }
5181           }
5182         }
5183
5184         PVE::Storage::activate_volumes($storecfg, $vollist);
5185
5186         if (!check_running($vmid, 1)) {
5187             eval {
5188                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5189                     outfunc => sub {}, errfunc => sub {});
5190             };
5191         }
5192
5193         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5194                                                   : $defaults->{cpuunits};
5195
5196         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5197         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5198
5199         my %properties = (
5200             Slice => 'qemu.slice',
5201             KillMode => 'none',
5202             CPUShares => $cpuunits
5203         );
5204
5205         if (my $cpulimit = $conf->{cpulimit}) {
5206             $properties{CPUQuota} = int($cpulimit * 100);
5207         }
5208         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5209
5210         my $run_qemu = sub {
5211             PVE::Tools::run_fork sub {
5212                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5213                 run_command($cmd, %run_params);
5214             };
5215         };
5216
5217         if ($conf->{hugepages}) {
5218
5219             my $code = sub {
5220                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5221                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5222
5223                 PVE::QemuServer::Memory::hugepages_mount();
5224                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5225
5226                 eval { $run_qemu->() };
5227                 if (my $err = $@) {
5228                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5229                     die $err;
5230                 }
5231
5232                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5233             };
5234             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5235
5236         } else {
5237             eval { $run_qemu->() };
5238         }
5239
5240         if (my $err = $@) {
5241             # deactivate volumes if start fails
5242             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5243             die "start failed: $err";
5244         }
5245
5246         print "migration listens on $migrate_uri\n" if $migrate_uri;
5247
5248         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5249             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5250             warn $@ if $@;
5251         }
5252
5253         #start nbd server for storage migration
5254         if ($targetstorage) {
5255             my $nodename = PVE::INotify::nodename();
5256             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5257             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5258             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5259             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5260
5261             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5262
5263             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5264
5265             foreach my $opt (sort keys %$local_volumes) {
5266                 my $volid = $local_volumes->{$opt};
5267                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5268                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5269                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5270             }
5271         }
5272
5273         if ($migratedfrom) {
5274             eval {
5275                 set_migration_caps($vmid);
5276             };
5277             warn $@ if $@;
5278
5279             if ($spice_port) {
5280                 print "spice listens on port $spice_port\n";
5281                 if ($spice_ticket) {
5282                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5283                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5284                 }
5285             }
5286
5287         } else {
5288             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5289                 if !$statefile && $conf->{balloon};
5290
5291             foreach my $opt (keys %$conf) {
5292                 next if $opt !~  m/^net\d+$/;
5293                 my $nicconf = parse_net($conf->{$opt});
5294                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5295             }
5296         }
5297
5298         vm_mon_cmd_nocheck($vmid, 'qom-set',
5299                     path => "machine/peripheral/balloon0",
5300                     property => "guest-stats-polling-interval",
5301                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5302
5303     });
5304 }
5305
5306 sub vm_mon_cmd {
5307     my ($vmid, $execute, %params) = @_;
5308
5309     my $cmd = { execute => $execute, arguments => \%params };
5310     vm_qmp_command($vmid, $cmd);
5311 }
5312
5313 sub vm_mon_cmd_nocheck {
5314     my ($vmid, $execute, %params) = @_;
5315
5316     my $cmd = { execute => $execute, arguments => \%params };
5317     vm_qmp_command($vmid, $cmd, 1);
5318 }
5319
5320 sub vm_qmp_command {
5321     my ($vmid, $cmd, $nocheck) = @_;
5322
5323     my $res;
5324
5325     my $timeout;
5326     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5327         $timeout = $cmd->{arguments}->{timeout};
5328         delete $cmd->{arguments}->{timeout};
5329     }
5330
5331     eval {
5332         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5333         my $sname = qmp_socket($vmid);
5334         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5335             my $qmpclient = PVE::QMPClient->new();
5336
5337             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5338         } else {
5339             die "unable to open monitor socket\n";
5340         }
5341     };
5342     if (my $err = $@) {
5343         syslog("err", "VM $vmid qmp command failed - $err");
5344         die $err;
5345     }
5346
5347     return $res;
5348 }
5349
5350 sub vm_human_monitor_command {
5351     my ($vmid, $cmdline) = @_;
5352
5353     my $res;
5354
5355     my $cmd = {
5356         execute => 'human-monitor-command',
5357         arguments => { 'command-line' => $cmdline},
5358     };
5359
5360     return vm_qmp_command($vmid, $cmd);
5361 }
5362
5363 sub vm_commandline {
5364     my ($storecfg, $vmid) = @_;
5365
5366     my $conf = PVE::QemuConfig->load_config($vmid);
5367
5368     my $defaults = load_defaults();
5369
5370     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5371
5372     return PVE::Tools::cmd2string($cmd);
5373 }
5374
5375 sub vm_reset {
5376     my ($vmid, $skiplock) = @_;
5377
5378     PVE::QemuConfig->lock_config($vmid, sub {
5379
5380         my $conf = PVE::QemuConfig->load_config($vmid);
5381
5382         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5383
5384         vm_mon_cmd($vmid, "system_reset");
5385     });
5386 }
5387
5388 sub get_vm_volumes {
5389     my ($conf) = @_;
5390
5391     my $vollist = [];
5392     foreach_volid($conf, sub {
5393         my ($volid, $attr) = @_;
5394
5395         return if $volid =~ m|^/|;
5396
5397         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5398         return if !$sid;
5399
5400         push @$vollist, $volid;
5401     });
5402
5403     return $vollist;
5404 }
5405
5406 sub vm_stop_cleanup {
5407     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5408
5409     eval {
5410
5411         if (!$keepActive) {
5412             my $vollist = get_vm_volumes($conf);
5413             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5414         }
5415
5416         foreach my $ext (qw(mon qmp pid vnc qga)) {
5417             unlink "/var/run/qemu-server/${vmid}.$ext";
5418         }
5419
5420         foreach my $key (keys %$conf) {
5421             next if $key !~ m/^hostpci(\d+)$/;
5422             my $hostpciindex = $1;
5423             my $d = parse_hostpci($conf->{$key});
5424             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5425
5426             foreach my $pci (@{$d->{pciid}}) {
5427                 my $pciid = $pci->{id};
5428                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5429             }
5430         }
5431
5432         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5433     };
5434     warn $@ if $@; # avoid errors - just warn
5435 }
5436
5437 # Note: use $nockeck to skip tests if VM configuration file exists.
5438 # We need that when migration VMs to other nodes (files already moved)
5439 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5440 sub vm_stop {
5441     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5442
5443     $force = 1 if !defined($force) && !$shutdown;
5444
5445     if ($migratedfrom){
5446         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5447         kill 15, $pid if $pid;
5448         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5449         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5450         return;
5451     }
5452
5453     PVE::QemuConfig->lock_config($vmid, sub {
5454
5455         my $pid = check_running($vmid, $nocheck);
5456         return if !$pid;
5457
5458         my $conf;
5459         if (!$nocheck) {
5460             $conf = PVE::QemuConfig->load_config($vmid);
5461             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5462             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5463                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5464                 $timeout = $opts->{down} if $opts->{down};
5465             }
5466         }
5467
5468         $timeout = 60 if !defined($timeout);
5469
5470         eval {
5471             if ($shutdown) {
5472                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5473                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5474                 } else {
5475                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5476                 }
5477             } else {
5478                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5479             }
5480         };
5481         my $err = $@;
5482
5483         if (!$err) {
5484             my $count = 0;
5485             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5486                 $count++;
5487                 sleep 1;
5488             }
5489
5490             if ($count >= $timeout) {
5491                 if ($force) {
5492                     warn "VM still running - terminating now with SIGTERM\n";
5493                     kill 15, $pid;
5494                 } else {
5495                     die "VM quit/powerdown failed - got timeout\n";
5496                 }
5497             } else {
5498                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5499                 return;
5500             }
5501         } else {
5502             if ($force) {
5503                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5504                 kill 15, $pid;
5505             } else {
5506                 die "VM quit/powerdown failed\n";
5507             }
5508         }
5509
5510         # wait again
5511         $timeout = 10;
5512
5513         my $count = 0;
5514         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5515             $count++;
5516             sleep 1;
5517         }
5518
5519         if ($count >= $timeout) {
5520             warn "VM still running - terminating now with SIGKILL\n";
5521             kill 9, $pid;
5522             sleep 1;
5523         }
5524
5525         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5526    });
5527 }
5528
5529 sub vm_suspend {
5530     my ($vmid, $skiplock) = @_;
5531
5532     PVE::QemuConfig->lock_config($vmid, sub {
5533
5534         my $conf = PVE::QemuConfig->load_config($vmid);
5535
5536         PVE::QemuConfig->check_lock($conf)
5537             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5538
5539         vm_mon_cmd($vmid, "stop");
5540     });
5541 }
5542
5543 sub vm_resume {
5544     my ($vmid, $skiplock, $nocheck) = @_;
5545
5546     PVE::QemuConfig->lock_config($vmid, sub {
5547
5548         my $res = vm_mon_cmd($vmid, 'query-status');
5549         my $resume_cmd = 'cont';
5550
5551         if ($res->{status} && $res->{status} eq 'suspended') {
5552             $resume_cmd = 'system_wakeup';
5553         }
5554
5555         if (!$nocheck) {
5556
5557             my $conf = PVE::QemuConfig->load_config($vmid);
5558
5559             PVE::QemuConfig->check_lock($conf)
5560                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5561
5562             vm_mon_cmd($vmid, $resume_cmd);
5563
5564         } else {
5565             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5566         }
5567     });
5568 }
5569
5570 sub vm_sendkey {
5571     my ($vmid, $skiplock, $key) = @_;
5572
5573     PVE::QemuConfig->lock_config($vmid, sub {
5574
5575         my $conf = PVE::QemuConfig->load_config($vmid);
5576
5577         # there is no qmp command, so we use the human monitor command
5578         vm_human_monitor_command($vmid, "sendkey $key");
5579     });
5580 }
5581
5582 sub vm_destroy {
5583     my ($storecfg, $vmid, $skiplock) = @_;
5584
5585     PVE::QemuConfig->lock_config($vmid, sub {
5586
5587         my $conf = PVE::QemuConfig->load_config($vmid);
5588
5589         if (!check_running($vmid)) {
5590             destroy_vm($storecfg, $vmid, undef, $skiplock);
5591         } else {
5592             die "VM $vmid is running - destroy failed\n";
5593         }
5594     });
5595 }
5596
5597 # vzdump restore implementaion
5598
5599 sub tar_archive_read_firstfile {
5600     my $archive = shift;
5601
5602     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5603
5604     # try to detect archive type first
5605     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5606         die "unable to open file '$archive'\n";
5607     my $firstfile = <$fh>;
5608     kill 15, $pid;
5609     close $fh;
5610
5611     die "ERROR: archive contaions no data\n" if !$firstfile;
5612     chomp $firstfile;
5613
5614     return $firstfile;
5615 }
5616
5617 sub tar_restore_cleanup {
5618     my ($storecfg, $statfile) = @_;
5619
5620     print STDERR "starting cleanup\n";
5621
5622     if (my $fd = IO::File->new($statfile, "r")) {
5623         while (defined(my $line = <$fd>)) {
5624             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5625                 my $volid = $2;
5626                 eval {
5627                     if ($volid =~ m|^/|) {
5628                         unlink $volid || die 'unlink failed\n';
5629                     } else {
5630                         PVE::Storage::vdisk_free($storecfg, $volid);
5631                     }
5632                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5633                 };
5634                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5635             } else {
5636                 print STDERR "unable to parse line in statfile - $line";
5637             }
5638         }
5639         $fd->close();
5640     }
5641 }
5642
5643 sub restore_archive {
5644     my ($archive, $vmid, $user, $opts) = @_;
5645
5646     my $format = $opts->{format};
5647     my $comp;
5648
5649     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5650         $format = 'tar' if !$format;
5651         $comp = 'gzip';
5652     } elsif ($archive =~ m/\.tar$/) {
5653         $format = 'tar' if !$format;
5654     } elsif ($archive =~ m/.tar.lzo$/) {
5655         $format = 'tar' if !$format;
5656         $comp = 'lzop';
5657     } elsif ($archive =~ m/\.vma$/) {
5658         $format = 'vma' if !$format;
5659     } elsif ($archive =~ m/\.vma\.gz$/) {
5660         $format = 'vma' if !$format;
5661         $comp = 'gzip';
5662     } elsif ($archive =~ m/\.vma\.lzo$/) {
5663         $format = 'vma' if !$format;
5664         $comp = 'lzop';
5665     } else {
5666         $format = 'vma' if !$format; # default
5667     }
5668
5669     # try to detect archive format
5670     if ($format eq 'tar') {
5671         return restore_tar_archive($archive, $vmid, $user, $opts);
5672     } else {
5673         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5674     }
5675 }
5676
5677 sub restore_update_config_line {
5678     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5679
5680     return if $line =~ m/^\#qmdump\#/;
5681     return if $line =~ m/^\#vzdump\#/;
5682     return if $line =~ m/^lock:/;
5683     return if $line =~ m/^unused\d+:/;
5684     return if $line =~ m/^parent:/;
5685     return if $line =~ m/^template:/; # restored VM is never a template
5686
5687     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5688     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5689         # try to convert old 1.X settings
5690         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5691         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5692             my ($model, $macaddr) = split(/\=/, $devconfig);
5693             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5694             my $net = {
5695                 model => $model,
5696                 bridge => "vmbr$ind",
5697                 macaddr => $macaddr,
5698             };
5699             my $netstr = print_net($net);
5700
5701             print $outfd "net$cookie->{netcount}: $netstr\n";
5702             $cookie->{netcount}++;
5703         }
5704     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5705         my ($id, $netstr) = ($1, $2);
5706         my $net = parse_net($netstr);
5707         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5708         $netstr = print_net($net);
5709         print $outfd "$id: $netstr\n";
5710     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5711         my $virtdev = $1;
5712         my $value = $3;
5713         my $di = parse_drive($virtdev, $value);
5714         if (defined($di->{backup}) && !$di->{backup}) {
5715             print $outfd "#$line";
5716         } elsif ($map->{$virtdev}) {
5717             delete $di->{format}; # format can change on restore
5718             $di->{file} = $map->{$virtdev};
5719             $value = print_drive($vmid, $di);
5720             print $outfd "$virtdev: $value\n";
5721         } else {
5722             print $outfd $line;
5723         }
5724     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5725         my $vmgenid = $1;
5726         if ($vmgenid ne '0') {
5727             # always generate a new vmgenid if there was a valid one setup
5728             $vmgenid = generate_uuid();
5729         }
5730         print $outfd "vmgenid: $vmgenid\n";
5731     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5732         my ($uuid, $uuid_str);
5733         UUID::generate($uuid);
5734         UUID::unparse($uuid, $uuid_str);
5735         my $smbios1 = parse_smbios1($2);
5736         $smbios1->{uuid} = $uuid_str;
5737         print $outfd $1.print_smbios1($smbios1)."\n";
5738     } else {
5739         print $outfd $line;
5740     }
5741 }
5742
5743 sub scan_volids {
5744     my ($cfg, $vmid) = @_;
5745
5746     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5747
5748     my $volid_hash = {};
5749     foreach my $storeid (keys %$info) {
5750         foreach my $item (@{$info->{$storeid}}) {
5751             next if !($item->{volid} && $item->{size});
5752             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5753             $volid_hash->{$item->{volid}} = $item;
5754         }
5755     }
5756
5757     return $volid_hash;
5758 }
5759
5760 sub is_volume_in_use {
5761     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5762
5763     my $path = PVE::Storage::path($storecfg, $volid);
5764
5765     my $scan_config = sub {
5766         my ($cref, $snapname) = @_;
5767
5768         foreach my $key (keys %$cref) {
5769             my $value = $cref->{$key};
5770             if (is_valid_drivename($key)) {
5771                 next if $skip_drive && $key eq $skip_drive;
5772                 my $drive = parse_drive($key, $value);
5773                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5774                 return 1 if $volid eq $drive->{file};
5775                 if ($drive->{file} =~ m!^/!) {
5776                     return 1 if $drive->{file} eq $path;
5777                 } else {
5778                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5779                     next if !$storeid;
5780                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5781                     next if !$scfg;
5782                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5783                 }
5784             }
5785         }
5786
5787         return 0;
5788     };
5789
5790     return 1 if &$scan_config($conf);
5791
5792     undef $skip_drive;
5793
5794     foreach my $snapname (keys %{$conf->{snapshots}}) {
5795         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5796     }
5797
5798     return 0;
5799 }
5800
5801 sub update_disksize {
5802     my ($vmid, $conf, $volid_hash) = @_;
5803
5804     my $changes;
5805     my $prefix = "VM $vmid:";
5806
5807     # used and unused disks
5808     my $referenced = {};
5809
5810     # Note: it is allowed to define multiple storages with same path (alias), so
5811     # we need to check both 'volid' and real 'path' (two different volid can point
5812     # to the same path).
5813
5814     my $referencedpath = {};
5815
5816     # update size info
5817     foreach my $opt (keys %$conf) {
5818         if (is_valid_drivename($opt)) {
5819             my $drive = parse_drive($opt, $conf->{$opt});
5820             my $volid = $drive->{file};
5821             next if !$volid;
5822
5823             $referenced->{$volid} = 1;
5824             if ($volid_hash->{$volid} &&
5825                 (my $path = $volid_hash->{$volid}->{path})) {
5826                 $referencedpath->{$path} = 1;
5827             }
5828
5829             next if drive_is_cdrom($drive);
5830             next if !$volid_hash->{$volid};
5831
5832             $drive->{size} = $volid_hash->{$volid}->{size};
5833             my $new = print_drive($vmid, $drive);
5834             if ($new ne $conf->{$opt}) {
5835                 $changes = 1;
5836                 $conf->{$opt} = $new;
5837                 print "$prefix update disk '$opt' information.\n";
5838             }
5839         }
5840     }
5841
5842     # remove 'unusedX' entry if volume is used
5843     foreach my $opt (keys %$conf) {
5844         next if $opt !~ m/^unused\d+$/;
5845         my $volid = $conf->{$opt};
5846         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5847         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5848             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5849             $changes = 1;
5850             delete $conf->{$opt};
5851         }
5852
5853         $referenced->{$volid} = 1;
5854         $referencedpath->{$path} = 1 if $path;
5855     }
5856
5857     foreach my $volid (sort keys %$volid_hash) {
5858         next if $volid =~ m/vm-$vmid-state-/;
5859         next if $referenced->{$volid};
5860         my $path = $volid_hash->{$volid}->{path};
5861         next if !$path; # just to be sure
5862         next if $referencedpath->{$path};
5863         $changes = 1;
5864         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5865         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5866         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5867     }
5868
5869     return $changes;
5870 }
5871
5872 sub rescan {
5873     my ($vmid, $nolock, $dryrun) = @_;
5874
5875     my $cfg = PVE::Storage::config();
5876
5877     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5878     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5879     foreach my $stor (keys %{$cfg->{ids}}) {
5880         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5881     }
5882
5883     print "rescan volumes...\n";
5884     my $volid_hash = scan_volids($cfg, $vmid);
5885
5886     my $updatefn =  sub {
5887         my ($vmid) = @_;
5888
5889         my $conf = PVE::QemuConfig->load_config($vmid);
5890
5891         PVE::QemuConfig->check_lock($conf);
5892
5893         my $vm_volids = {};
5894         foreach my $volid (keys %$volid_hash) {
5895             my $info = $volid_hash->{$volid};
5896             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5897         }
5898
5899         my $changes = update_disksize($vmid, $conf, $vm_volids);
5900
5901         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5902     };
5903
5904     if (defined($vmid)) {
5905         if ($nolock) {
5906             &$updatefn($vmid);
5907         } else {
5908             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5909         }
5910     } else {
5911         my $vmlist = config_list();
5912         foreach my $vmid (keys %$vmlist) {
5913             if ($nolock) {
5914                 &$updatefn($vmid);
5915             } else {
5916                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5917             }
5918         }
5919     }
5920 }
5921
5922 sub restore_vma_archive {
5923     my ($archive, $vmid, $user, $opts, $comp) = @_;
5924
5925     my $readfrom = $archive;
5926
5927     my $cfg = PVE::Storage::config();
5928     my $commands = [];
5929     my $bwlimit = $opts->{bwlimit};
5930
5931     my $dbg_cmdstring = '';
5932     my $add_pipe = sub {
5933         my ($cmd) = @_;
5934         push @$commands, $cmd;
5935         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5936         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5937         $readfrom = '-';
5938     };
5939
5940     my $input = undef;
5941     if ($archive eq '-') {
5942         $input = '<&STDIN';
5943     } else {
5944         # If we use a backup from a PVE defined storage we also consider that
5945         # storage's rate limit:
5946         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5947         if (defined($volid)) {
5948             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5949             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5950             if ($readlimit) {
5951                 print STDERR "applying read rate limit: $readlimit\n";
5952                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5953                 $add_pipe->($cstream);
5954             }
5955         }
5956     }
5957
5958     if ($comp) {
5959         my $cmd;
5960         if ($comp eq 'gzip') {
5961             $cmd = ['zcat', $readfrom];
5962         } elsif ($comp eq 'lzop') {
5963             $cmd = ['lzop', '-d', '-c', $readfrom];
5964         } else {
5965             die "unknown compression method '$comp'\n";
5966         }
5967         $add_pipe->($cmd);
5968     }
5969
5970     my $tmpdir = "/var/tmp/vzdumptmp$$";
5971     rmtree $tmpdir;
5972
5973     # disable interrupts (always do cleanups)
5974     local $SIG{INT} =
5975         local $SIG{TERM} =
5976         local $SIG{QUIT} =
5977         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5978
5979     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5980     POSIX::mkfifo($mapfifo, 0600);
5981     my $fifofh;
5982
5983     my $openfifo = sub {
5984         open($fifofh, '>', $mapfifo) || die $!;
5985     };
5986
5987     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5988
5989     my $oldtimeout;
5990     my $timeout = 5;
5991
5992     my $devinfo = {};
5993
5994     my $rpcenv = PVE::RPCEnvironment::get();
5995
5996     my $conffile = PVE::QemuConfig->config_file($vmid);
5997     my $tmpfn = "$conffile.$$.tmp";
5998
5999     # Note: $oldconf is undef if VM does not exists
6000     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6001     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6002
6003     my %storage_limits;
6004
6005     my $print_devmap = sub {
6006         my $virtdev_hash = {};
6007
6008         my $cfgfn = "$tmpdir/qemu-server.conf";
6009
6010         # we can read the config - that is already extracted
6011         my $fh = IO::File->new($cfgfn, "r") ||
6012             "unable to read qemu-server.conf - $!\n";
6013
6014         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6015         if (-f $fwcfgfn) {
6016             my $pve_firewall_dir = '/etc/pve/firewall';
6017             mkdir $pve_firewall_dir; # make sure the dir exists
6018             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6019         }
6020
6021         while (defined(my $line = <$fh>)) {
6022             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6023                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6024                 die "archive does not contain data for drive '$virtdev'\n"
6025                     if !$devinfo->{$devname};
6026                 if (defined($opts->{storage})) {
6027                     $storeid = $opts->{storage} || 'local';
6028                 } elsif (!$storeid) {
6029                     $storeid = 'local';
6030                 }
6031                 $format = 'raw' if !$format;
6032                 $devinfo->{$devname}->{devname} = $devname;
6033                 $devinfo->{$devname}->{virtdev} = $virtdev;
6034                 $devinfo->{$devname}->{format} = $format;
6035                 $devinfo->{$devname}->{storeid} = $storeid;
6036
6037                 # check permission on storage
6038                 my $pool = $opts->{pool}; # todo: do we need that?
6039                 if ($user ne 'root@pam') {
6040                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6041                 }
6042
6043                 $storage_limits{$storeid} = $bwlimit;
6044
6045                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6046             }
6047         }
6048
6049         foreach my $key (keys %storage_limits) {
6050             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6051             next if !$limit;
6052             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6053             $storage_limits{$key} = $limit * 1024;
6054         }
6055
6056         foreach my $devname (keys %$devinfo) {
6057             die "found no device mapping information for device '$devname'\n"
6058                 if !$devinfo->{$devname}->{virtdev};
6059         }
6060
6061         # create empty/temp config
6062         if ($oldconf) {
6063             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6064             foreach_drive($oldconf, sub {
6065                 my ($ds, $drive) = @_;
6066
6067                 return if drive_is_cdrom($drive);
6068
6069                 my $volid = $drive->{file};
6070
6071                 return if !$volid || $volid =~ m|^/|;
6072
6073                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6074                 return if !$path || !$owner || ($owner != $vmid);
6075
6076                 # Note: only delete disk we want to restore
6077                 # other volumes will become unused
6078                 if ($virtdev_hash->{$ds}) {
6079                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6080                     if (my $err = $@) {
6081                         warn $err;
6082                     }
6083                 }
6084             });
6085
6086             # delete vmstate files
6087             # since after the restore we have no snapshots anymore
6088             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6089                 my $snap = $oldconf->{snapshots}->{$snapname};
6090                 if ($snap->{vmstate}) {
6091                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6092                     if (my $err = $@) {
6093                         warn $err;
6094                     }
6095                 }
6096             }
6097         }
6098
6099         my $map = {};
6100         foreach my $virtdev (sort keys %$virtdev_hash) {
6101             my $d = $virtdev_hash->{$virtdev};
6102             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6103             my $storeid = $d->{storeid};
6104             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6105
6106             my $map_opts = '';
6107             if (my $limit = $storage_limits{$storeid}) {
6108                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6109             }
6110
6111             # test if requested format is supported
6112             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6113             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6114             $d->{format} = $defFormat if !$supported;
6115
6116             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6117                                                   $d->{format}, undef, $alloc_size);
6118             print STDERR "new volume ID is '$volid'\n";
6119             $d->{volid} = $volid;
6120             my $path = PVE::Storage::path($cfg, $volid);
6121
6122             PVE::Storage::activate_volumes($cfg,[$volid]);
6123
6124             my $write_zeros = 1;
6125             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6126                 $write_zeros = 0;
6127             }
6128
6129             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6130
6131             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6132             $map->{$virtdev} = $volid;
6133         }
6134
6135         $fh->seek(0, 0) || die "seek failed - $!\n";
6136
6137         my $outfd = new IO::File ($tmpfn, "w") ||
6138             die "unable to write config for VM $vmid\n";
6139
6140         my $cookie = { netcount => 0 };
6141         while (defined(my $line = <$fh>)) {
6142             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6143         }
6144
6145         $fh->close();
6146         $outfd->close();
6147     };
6148
6149     eval {
6150         # enable interrupts
6151         local $SIG{INT} =
6152             local $SIG{TERM} =
6153             local $SIG{QUIT} =
6154             local $SIG{HUP} =
6155             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6156         local $SIG{ALRM} = sub { die "got timeout\n"; };
6157
6158         $oldtimeout = alarm($timeout);
6159
6160         my $parser = sub {
6161             my $line = shift;
6162
6163             print "$line\n";
6164
6165             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6166                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6167                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6168             } elsif ($line =~ m/^CTIME: /) {
6169                 # we correctly received the vma config, so we can disable
6170                 # the timeout now for disk allocation (set to 10 minutes, so
6171                 # that we always timeout if something goes wrong)
6172                 alarm(600);
6173                 &$print_devmap();
6174                 print $fifofh "done\n";
6175                 my $tmp = $oldtimeout || 0;
6176                 $oldtimeout = undef;
6177                 alarm($tmp);
6178                 close($fifofh);
6179             }
6180         };
6181
6182         print "restore vma archive: $dbg_cmdstring\n";
6183         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6184     };
6185     my $err = $@;
6186
6187     alarm($oldtimeout) if $oldtimeout;
6188
6189     my $vollist = [];
6190     foreach my $devname (keys %$devinfo) {
6191         my $volid = $devinfo->{$devname}->{volid};
6192         push @$vollist, $volid if $volid;
6193     }
6194
6195     PVE::Storage::deactivate_volumes($cfg, $vollist);
6196
6197     unlink $mapfifo;
6198
6199     if ($err) {
6200         rmtree $tmpdir;
6201         unlink $tmpfn;
6202
6203         foreach my $devname (keys %$devinfo) {
6204             my $volid = $devinfo->{$devname}->{volid};
6205             next if !$volid;
6206             eval {
6207                 if ($volid =~ m|^/|) {
6208                     unlink $volid || die 'unlink failed\n';
6209                 } else {
6210                     PVE::Storage::vdisk_free($cfg, $volid);
6211                 }
6212                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6213             };
6214             print STDERR "unable to cleanup '$volid' - $@" if $@;
6215         }
6216         die $err;
6217     }
6218
6219     rmtree $tmpdir;
6220
6221     rename($tmpfn, $conffile) ||
6222         die "unable to commit configuration file '$conffile'\n";
6223
6224     PVE::Cluster::cfs_update(); # make sure we read new file
6225
6226     eval { rescan($vmid, 1); };
6227     warn $@ if $@;
6228 }
6229
6230 sub restore_tar_archive {
6231     my ($archive, $vmid, $user, $opts) = @_;
6232
6233     if ($archive ne '-') {
6234         my $firstfile = tar_archive_read_firstfile($archive);
6235         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6236             if $firstfile ne 'qemu-server.conf';
6237     }
6238
6239     my $storecfg = PVE::Storage::config();
6240
6241     # destroy existing data - keep empty config
6242     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6243     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6244
6245     my $tocmd = "/usr/lib/qemu-server/qmextract";
6246
6247     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6248     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6249     $tocmd .= ' --prealloc' if $opts->{prealloc};
6250     $tocmd .= ' --info' if $opts->{info};
6251
6252     # tar option "xf" does not autodetect compression when read from STDIN,
6253     # so we pipe to zcat
6254     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6255         PVE::Tools::shellquote("--to-command=$tocmd");
6256
6257     my $tmpdir = "/var/tmp/vzdumptmp$$";
6258     mkpath $tmpdir;
6259
6260     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6261     local $ENV{VZDUMP_VMID} = $vmid;
6262     local $ENV{VZDUMP_USER} = $user;
6263
6264     my $conffile = PVE::QemuConfig->config_file($vmid);
6265     my $tmpfn = "$conffile.$$.tmp";
6266
6267     # disable interrupts (always do cleanups)
6268     local $SIG{INT} =
6269         local $SIG{TERM} =
6270         local $SIG{QUIT} =
6271         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6272
6273     eval {
6274         # enable interrupts
6275         local $SIG{INT} =
6276             local $SIG{TERM} =
6277             local $SIG{QUIT} =
6278             local $SIG{HUP} =
6279             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6280
6281         if ($archive eq '-') {
6282             print "extracting archive from STDIN\n";
6283             run_command($cmd, input => "<&STDIN");
6284         } else {
6285             print "extracting archive '$archive'\n";
6286             run_command($cmd);
6287         }
6288
6289         return if $opts->{info};
6290
6291         # read new mapping
6292         my $map = {};
6293         my $statfile = "$tmpdir/qmrestore.stat";
6294         if (my $fd = IO::File->new($statfile, "r")) {
6295             while (defined (my $line = <$fd>)) {
6296                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6297                     $map->{$1} = $2 if $1;
6298                 } else {
6299                     print STDERR "unable to parse line in statfile - $line\n";
6300                 }
6301             }
6302             $fd->close();
6303         }
6304
6305         my $confsrc = "$tmpdir/qemu-server.conf";
6306
6307         my $srcfd = new IO::File($confsrc, "r") ||
6308             die "unable to open file '$confsrc'\n";
6309
6310         my $outfd = new IO::File ($tmpfn, "w") ||
6311             die "unable to write config for VM $vmid\n";
6312
6313         my $cookie = { netcount => 0 };
6314         while (defined (my $line = <$srcfd>)) {
6315             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6316         }
6317
6318         $srcfd->close();
6319         $outfd->close();
6320     };
6321     my $err = $@;
6322
6323     if ($err) {
6324
6325         unlink $tmpfn;
6326
6327         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6328
6329         die $err;
6330     }
6331
6332     rmtree $tmpdir;
6333
6334     rename $tmpfn, $conffile ||
6335         die "unable to commit configuration file '$conffile'\n";
6336
6337     PVE::Cluster::cfs_update(); # make sure we read new file
6338
6339     eval { rescan($vmid, 1); };
6340     warn $@ if $@;
6341 };
6342
6343 sub foreach_storage_used_by_vm {
6344     my ($conf, $func) = @_;
6345
6346     my $sidhash = {};
6347
6348     foreach_drive($conf, sub {
6349         my ($ds, $drive) = @_;
6350         return if drive_is_cdrom($drive);
6351
6352         my $volid = $drive->{file};
6353
6354         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6355         $sidhash->{$sid} = $sid if $sid;
6356     });
6357
6358     foreach my $sid (sort keys %$sidhash) {
6359         &$func($sid);
6360     }
6361 }
6362
6363 sub do_snapshots_with_qemu {
6364     my ($storecfg, $volid) = @_;
6365
6366     my $storage_name = PVE::Storage::parse_volume_id($volid);
6367
6368     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6369         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6370         return 1;
6371     }
6372
6373     if ($volid =~ m/\.(qcow2|qed)$/){
6374         return 1;
6375     }
6376
6377     return undef;
6378 }
6379
6380 sub qga_check_running {
6381     my ($vmid, $nowarn) = @_;
6382
6383     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6384     if ($@) {
6385         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6386         return 0;
6387     }
6388     return 1;
6389 }
6390
6391 sub template_create {
6392     my ($vmid, $conf, $disk) = @_;
6393
6394     my $storecfg = PVE::Storage::config();
6395
6396     foreach_drive($conf, sub {
6397         my ($ds, $drive) = @_;
6398
6399         return if drive_is_cdrom($drive);
6400         return if $disk && $ds ne $disk;
6401
6402         my $volid = $drive->{file};
6403         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6404
6405         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6406         $drive->{file} = $voliddst;
6407         $conf->{$ds} = print_drive($vmid, $drive);
6408         PVE::QemuConfig->write_config($vmid, $conf);
6409     });
6410 }
6411
6412 sub qemu_img_convert {
6413     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6414
6415     my $storecfg = PVE::Storage::config();
6416     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6417     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6418
6419     if ($src_storeid && $dst_storeid) {
6420
6421         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6422
6423         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6424         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6425
6426         my $src_format = qemu_img_format($src_scfg, $src_volname);
6427         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6428
6429         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6430         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6431
6432         my $cmd = [];
6433         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6434         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6435         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6436         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6437         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6438         if ($is_zero_initialized) {
6439             push @$cmd, "zeroinit:$dst_path";
6440         } else {
6441             push @$cmd, $dst_path;
6442         }
6443
6444         my $parser = sub {
6445             my $line = shift;
6446             if($line =~ m/\((\S+)\/100\%\)/){
6447                 my $percent = $1;
6448                 my $transferred = int($size * $percent / 100);
6449                 my $remaining = $size - $transferred;
6450
6451                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6452             }
6453
6454         };
6455
6456         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6457         my $err = $@;
6458         die "copy failed: $err" if $err;
6459     }
6460 }
6461
6462 sub qemu_img_format {
6463     my ($scfg, $volname) = @_;
6464
6465     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6466         return $1;
6467     } else {
6468         return "raw";
6469     }
6470 }
6471
6472 sub qemu_drive_mirror {
6473     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6474
6475     $jobs = {} if !$jobs;
6476
6477     my $qemu_target;
6478     my $format;
6479     $jobs->{"drive-$drive"} = {};
6480
6481     if ($dst_volid =~ /^nbd:/) {
6482         $qemu_target = $dst_volid;
6483         $format = "nbd";
6484     } else {
6485         my $storecfg = PVE::Storage::config();
6486         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6487
6488         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6489
6490         $format = qemu_img_format($dst_scfg, $dst_volname);
6491
6492         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6493
6494         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6495     }
6496
6497     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6498     $opts->{format} = $format if $format;
6499
6500     print "drive mirror is starting for drive-$drive\n";
6501
6502     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6503
6504     if (my $err = $@) {
6505         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6506         die "mirroring error: $err";
6507     }
6508
6509     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6510 }
6511
6512 sub qemu_drive_mirror_monitor {
6513     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6514
6515     eval {
6516         my $err_complete = 0;
6517
6518         while (1) {
6519             die "storage migration timed out\n" if $err_complete > 300;
6520
6521             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6522
6523             my $running_mirror_jobs = {};
6524             foreach my $stat (@$stats) {
6525                 next if $stat->{type} ne 'mirror';
6526                 $running_mirror_jobs->{$stat->{device}} = $stat;
6527             }
6528
6529             my $readycounter = 0;
6530
6531             foreach my $job (keys %$jobs) {
6532
6533                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6534                     print "$job : finished\n";
6535                     delete $jobs->{$job};
6536                     next;
6537                 }
6538
6539                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6540
6541                 my $busy = $running_mirror_jobs->{$job}->{busy};
6542                 my $ready = $running_mirror_jobs->{$job}->{ready};
6543                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6544                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6545                     my $remaining = $total - $transferred;
6546                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6547
6548                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6549                 }
6550
6551                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6552             }
6553
6554             last if scalar(keys %$jobs) == 0;
6555
6556             if ($readycounter == scalar(keys %$jobs)) {
6557                 print "all mirroring jobs are ready \n";
6558                 last if $skipcomplete; #do the complete later
6559
6560                 if ($vmiddst && $vmiddst != $vmid) {
6561                     my $agent_running = $qga && qga_check_running($vmid);
6562                     if ($agent_running) {
6563                         print "freeze filesystem\n";
6564                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6565                     } else {
6566                         print "suspend vm\n";
6567                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6568                     }
6569
6570                     # if we clone a disk for a new target vm, we don't switch the disk
6571                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6572
6573                     if ($agent_running) {
6574                         print "unfreeze filesystem\n";
6575                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6576                     } else {
6577                         print "resume vm\n";
6578                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6579                     }
6580
6581                     last;
6582                 } else {
6583
6584                     foreach my $job (keys %$jobs) {
6585                         # try to switch the disk if source and destination are on the same guest
6586                         print "$job: Completing block job...\n";
6587
6588                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6589                         if ($@ =~ m/cannot be completed/) {
6590                             print "$job: Block job cannot be completed, try again.\n";
6591                             $err_complete++;
6592                         }else {
6593                             print "$job: Completed successfully.\n";
6594                             $jobs->{$job}->{complete} = 1;
6595                         }
6596                     }
6597                 }
6598             }
6599             sleep 1;
6600         }
6601     };
6602     my $err = $@;
6603
6604     if ($err) {
6605         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6606         die "mirroring error: $err";
6607     }
6608
6609 }
6610
6611 sub qemu_blockjobs_cancel {
6612     my ($vmid, $jobs) = @_;
6613
6614     foreach my $job (keys %$jobs) {
6615         print "$job: Cancelling block job\n";
6616         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6617         $jobs->{$job}->{cancel} = 1;
6618     }
6619
6620     while (1) {
6621         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6622
6623         my $running_jobs = {};
6624         foreach my $stat (@$stats) {
6625             $running_jobs->{$stat->{device}} = $stat;
6626         }
6627
6628         foreach my $job (keys %$jobs) {
6629
6630             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6631                 print "$job: Done.\n";
6632                 delete $jobs->{$job};
6633             }
6634         }
6635
6636         last if scalar(keys %$jobs) == 0;
6637
6638         sleep 1;
6639     }
6640 }
6641
6642 sub clone_disk {
6643     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6644         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6645
6646     my $newvolid;
6647
6648     if (!$full) {
6649         print "create linked clone of drive $drivename ($drive->{file})\n";
6650         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6651         push @$newvollist, $newvolid;
6652     } else {
6653
6654         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6655         $storeid = $storage if $storage;
6656
6657         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6658         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6659
6660         print "create full clone of drive $drivename ($drive->{file})\n";
6661         my $name = undef;
6662         if (drive_is_cloudinit($drive)) {
6663             $name = "vm-$newvmid-cloudinit";
6664             $snapname = undef;
6665             # cloudinit only supports raw and qcow2 atm:
6666             if ($dst_format eq 'qcow2') {
6667                 $name .= '.qcow2';
6668             } elsif ($dst_format ne 'raw') {
6669                 die "clone: unhandled format for cloudinit image\n";
6670             }
6671         }
6672         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6673         push @$newvollist, $newvolid;
6674
6675         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6676
6677         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6678         if (!$running || $snapname) {
6679             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6680         } else {
6681
6682             my $kvmver = get_running_qemu_version ($vmid);
6683             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6684                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6685                     if $drive->{iothread};
6686             }
6687
6688             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6689         }
6690     }
6691
6692     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6693
6694     my $disk = $drive;
6695     $disk->{format} = undef;
6696     $disk->{file} = $newvolid;
6697     $disk->{size} = $size;
6698
6699     return $disk;
6700 }
6701
6702 # this only works if VM is running
6703 sub get_current_qemu_machine {
6704     my ($vmid) = @_;
6705
6706     my $cmd = { execute => 'query-machines', arguments => {} };
6707     my $res = vm_qmp_command($vmid, $cmd);
6708
6709     my ($current, $default);
6710     foreach my $e (@$res) {
6711         $default = $e->{name} if $e->{'is-default'};
6712         $current = $e->{name} if $e->{'is-current'};
6713     }
6714
6715     # fallback to the default machine if current is not supported by qemu
6716     return $current || $default || 'pc';
6717 }
6718
6719 sub get_running_qemu_version {
6720     my ($vmid) = @_;
6721     my $cmd = { execute => 'query-version', arguments => {} };
6722     my $res = vm_qmp_command($vmid, $cmd);
6723     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6724 }
6725
6726 sub qemu_machine_feature_enabled {
6727     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6728
6729     my $current_major;
6730     my $current_minor;
6731
6732     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6733
6734         $current_major = $3;
6735         $current_minor = $4;
6736
6737     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6738
6739         $current_major = $1;
6740         $current_minor = $2;
6741     }
6742
6743     return 1 if $current_major > $version_major ||
6744                 ($current_major == $version_major &&
6745                  $current_minor >= $version_minor);
6746 }
6747
6748 sub qemu_machine_pxe {
6749     my ($vmid, $conf, $machine) = @_;
6750
6751     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6752
6753     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6754         $machine .= '.pxe';
6755     }
6756
6757     return $machine;
6758 }
6759
6760 sub qemu_use_old_bios_files {
6761     my ($machine_type) = @_;
6762
6763     return if !$machine_type;
6764
6765     my $use_old_bios_files = undef;
6766
6767     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6768         $machine_type = $1;
6769         $use_old_bios_files = 1;
6770     } else {
6771         my $kvmver = kvm_user_version();
6772         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6773         # load new efi bios files on migration. So this hack is required to allow
6774         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6775         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6776         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6777     }
6778
6779     return ($use_old_bios_files, $machine_type);
6780 }
6781
6782 sub create_efidisk($$$$$) {
6783     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6784
6785     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6786     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6787
6788     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6789     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6790     PVE::Storage::activate_volumes($storecfg, [$volid]);
6791
6792     my $path = PVE::Storage::path($storecfg, $volid);
6793     eval {
6794         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6795     };
6796     die "Copying EFI vars image failed: $@" if $@;
6797
6798     return ($volid, $vars_size);
6799 }
6800
6801 sub vm_iothreads_list {
6802     my ($vmid) = @_;
6803
6804     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6805
6806     my $iothreads = {};
6807     foreach my $iothread (@$res) {
6808         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6809     }
6810
6811     return $iothreads;
6812 }
6813
6814 sub scsihw_infos {
6815     my ($conf, $drive) = @_;
6816
6817     my $maxdev = 0;
6818
6819     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6820         $maxdev = 7;
6821     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6822         $maxdev = 1;
6823     } else {
6824         $maxdev = 256;
6825     }
6826
6827     my $controller = int($drive->{index} / $maxdev);
6828     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6829
6830     return ($maxdev, $controller, $controller_prefix);
6831 }
6832
6833 sub add_hyperv_enlightenments {
6834     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6835
6836     return if $winversion < 6;
6837     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6838
6839     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6840
6841     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6842         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6843         push @$cpuFlags , 'hv_vapic';
6844         push @$cpuFlags , 'hv_time';
6845     } else {
6846         push @$cpuFlags , 'hv_spinlocks=0xffff';
6847     }
6848
6849     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6850         push @$cpuFlags , 'hv_reset';
6851         push @$cpuFlags , 'hv_vpindex';
6852         push @$cpuFlags , 'hv_runtime';
6853     }
6854
6855     if ($winversion >= 7) {
6856         push @$cpuFlags , 'hv_relaxed';
6857
6858         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6859             push @$cpuFlags , 'hv_synic';
6860             push @$cpuFlags , 'hv_stimer';
6861         }
6862     }
6863 }
6864
6865 sub windows_version {
6866     my ($ostype) = @_;
6867
6868     return 0 if !$ostype;
6869
6870     my $winversion = 0;
6871
6872     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6873         $winversion = 5;
6874     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6875         $winversion = 6;
6876     } elsif ($ostype =~ m/^win(\d+)$/) {
6877         $winversion = $1;
6878     }
6879
6880     return $winversion;
6881 }
6882
6883 sub resolve_dst_disk_format {
6884         my ($storecfg, $storeid, $src_volname, $format) = @_;
6885         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6886
6887         if (!$format) {
6888             # if no target format is specified, use the source disk format as hint
6889             if ($src_volname) {
6890                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6891                 $format = qemu_img_format($scfg, $src_volname);
6892             } else {
6893                 return $defFormat;
6894             }
6895         }
6896
6897         # test if requested format is supported - else use default
6898         my $supported = grep { $_ eq $format } @$validFormats;
6899         $format = $defFormat if !$supported;
6900         return $format;
6901 }
6902
6903 sub resolve_first_disk {
6904     my $conf = shift;
6905     my @disks = PVE::QemuServer::valid_drive_names();
6906     my $firstdisk;
6907     foreach my $ds (reverse @disks) {
6908         next if !$conf->{$ds};
6909         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6910         next if PVE::QemuServer::drive_is_cdrom($disk);
6911         $firstdisk = $ds;
6912     }
6913     return $firstdisk;
6914 }
6915
6916 sub generate_uuid {
6917     my ($uuid, $uuid_str);
6918     UUID::generate($uuid);
6919     UUID::unparse($uuid, $uuid_str);
6920     return $uuid_str;
6921 }
6922
6923 sub generate_smbios1_uuid {
6924     return "uuid=".generate_uuid();
6925 }
6926
6927 sub nbd_stop {
6928     my ($vmid) = @_;
6929
6930     vm_mon_cmd($vmid, 'nbd-server-stop');
6931 }
6932
6933 # bash completion helper
6934
6935 sub complete_backup_archives {
6936     my ($cmdname, $pname, $cvalue) = @_;
6937
6938     my $cfg = PVE::Storage::config();
6939
6940     my $storeid;
6941
6942     if ($cvalue =~ m/^([^:]+):/) {
6943         $storeid = $1;
6944     }
6945
6946     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6947
6948     my $res = [];
6949     foreach my $id (keys %$data) {
6950         foreach my $item (@{$data->{$id}}) {
6951             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6952             push @$res, $item->{volid} if defined($item->{volid});
6953         }
6954     }
6955
6956     return $res;
6957 }
6958
6959 my $complete_vmid_full = sub {
6960     my ($running) = @_;
6961
6962     my $idlist = vmstatus();
6963
6964     my $res = [];
6965
6966     foreach my $id (keys %$idlist) {
6967         my $d = $idlist->{$id};
6968         if (defined($running)) {
6969             next if $d->{template};
6970             next if $running && $d->{status} ne 'running';
6971             next if !$running && $d->{status} eq 'running';
6972         }
6973         push @$res, $id;
6974
6975     }
6976     return $res;
6977 };
6978
6979 sub complete_vmid {
6980     return &$complete_vmid_full();
6981 }
6982
6983 sub complete_vmid_stopped {
6984     return &$complete_vmid_full(0);
6985 }
6986
6987 sub complete_vmid_running {
6988     return &$complete_vmid_full(1);
6989 }
6990
6991 sub complete_storage {
6992
6993     my $cfg = PVE::Storage::config();
6994     my $ids = $cfg->{ids};
6995
6996     my $res = [];
6997     foreach my $sid (keys %$ids) {
6998         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6999         next if !$ids->{$sid}->{content}->{images};
7000         push @$res, $sid;
7001     }
7002
7003     return $res;
7004 }
7005
7006 1;