PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $vga_fmt = {
 227     type => {
 228         description => "Select the VGA type.",
 229         type => 'string',
 230         default => 'std',
 231         optional => 1,
 232         default_key => 1,
 233         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 234     },
 235     memory => {
 236         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 237         type => 'integer',
 238         optional => 1,
 239         minimum => 4,
 240         maximum => 512,
 241     },
 242 };
 243
 244 my $confdesc = {
 245     onboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Specifies whether a VM will be started during system bootup.",
 249         default => 0,
 250     },
 251     autostart => {
 252         optional => 1,
 253         type => 'boolean',
 254         description => "Automatic restart after crash (currently ignored).",
 255         default => 0,
 256     },
 257     hotplug => {
 258         optional => 1,
 259         type => 'string', format => 'pve-hotplug-features',
 260         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 261         default => 'network,disk,usb',
 262     },
 263     reboot => {
 264         optional => 1,
 265         type => 'boolean',
 266         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 267         default => 1,
 268     },
 269     lock => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Lock/unlock the VM.",
 273         enum => [qw(migrate backup snapshot rollback)],
 274     },
 275     cpulimit => {
 276         optional => 1,
 277         type => 'number',
 278         description => "Limit of CPU usage.",
 279         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 280         minimum => 0,
 281         maximum => 128,
 282         default => 0,
 283     },
 284     cpuunits => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "CPU weight for a VM.",
 288         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 289         minimum => 2,
 290         maximum => 262144,
 291         default => 1024,
 292     },
 293     memory => {
 294         optional => 1,
 295         type => 'integer',
 296         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 297         minimum => 16,
 298         default => 512,
 299     },
 300     balloon => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 304         minimum => 0,
 305     },
 306     shares => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 310         minimum => 0,
 311         maximum => 50000,
 312         default => 1000,
 313     },
 314     keyboard => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 318                        "It should not be necessary to set it.",
 319         enum => PVE::Tools::kvmkeymaplist(),
 320         default => undef,
 321     },
 322     name => {
 323         optional => 1,
 324         type => 'string', format => 'dns-name',
 325         description => "Set a name for the VM. Only used on the configuration web interface.",
 326     },
 327     scsihw => {
 328         optional => 1,
 329         type => 'string',
 330         description => "SCSI controller model",
 331         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 332         default => 'lsi',
 333     },
 334     description => {
 335         optional => 1,
 336         type => 'string',
 337         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 338     },
 339     ostype => {
 340         optional => 1,
 341         type => 'string',
 342         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 343         description => "Specify guest operating system.",
 344         verbose_description => <<EODESC,
 345 Specify guest operating system. This is used to enable special
 346 optimization/features for specific operating systems:
 347
 348 [horizontal]
 349 other;; unspecified OS
 350 wxp;; Microsoft Windows XP
 351 w2k;; Microsoft Windows 2000
 352 w2k3;; Microsoft Windows 2003
 353 w2k8;; Microsoft Windows 2008
 354 wvista;; Microsoft Windows Vista
 355 win7;; Microsoft Windows 7
 356 win8;; Microsoft Windows 8/2012/2012r2
 357 win10;; Microsoft Windows 10/2016
 358 l24;; Linux 2.4 Kernel
 359 l26;; Linux 2.6/3.X Kernel
 360 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 361 EODESC
 362     },
 363     boot => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 367         pattern => '[acdn]{1,4}',
 368         default => 'cdn',
 369     },
 370     bootdisk => {
 371         optional => 1,
 372         type => 'string', format => 'pve-qm-bootdisk',
 373         description => "Enable booting from specified disk.",
 374         pattern => '(ide|sata|scsi|virtio)\d+',
 375     },
 376     smp => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "The number of CPUs. Please use option -sockets instead.",
 380         minimum => 1,
 381         default => 1,
 382     },
 383     sockets => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPU sockets.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     cores => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of cores per socket.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     numa => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Enable/disable NUMA.",
 401         default => 0,
 402     },
 403     hugepages => {
 404         optional => 1,
 405         type => 'string',
 406         description => "Enable/disable hugepages memory.",
 407         enum => [qw(any 2 1024)],
 408     },
 409     vcpus => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "Number of hotplugged vcpus.",
 413         minimum => 1,
 414         default => 0,
 415     },
 416     acpi => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable ACPI.",
 420         default => 1,
 421     },
 422     agent => {
 423         optional => 1,
 424         description => "Enable/disable Qemu GuestAgent and its properties.",
 425         type => 'string',
 426         format => $agent_fmt,
 427     },
 428     kvm => {
 429         optional => 1,
 430         type => 'boolean',
 431         description => "Enable/disable KVM hardware virtualization.",
 432         default => 1,
 433     },
 434     tdf => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable time drift fix.",
 438         default => 0,
 439     },
 440     localtime => {
 441         optional => 1,
 442         type => 'boolean',
 443         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 444     },
 445     freeze => {
 446         optional => 1,
 447         type => 'boolean',
 448         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 449     },
 450     vga => {
 451         optional => 1,
 452         type => 'string', format => $vga_fmt,
 453         description => "Configure the VGA hardware.",
 454         verbose_description => "Configure the VGA Hardware. If you want to use ".
 455             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 456             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 457             "is 'std' for all OS types besides some Windows versions (XP and " .
 458             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 459             "display server. For win* OS you can select how many independent " .
 460             "displays you want, Linux guests can add displays them self.\n".
 461             "You can also run without any graphic card, using a serial device as terminal.",
 462     },
 463     watchdog => {
 464         optional => 1,
 465         type => 'string', format => 'pve-qm-watchdog',
 466         description => "Create a virtual hardware watchdog device.",
 467         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 468             " (by a guest action), the watchdog must be periodically polled " .
 469             "by an agent inside the guest or else the watchdog will reset " .
 470             "the guest (or execute the respective action specified)",
 471     },
 472     startdate => {
 473         optional => 1,
 474         type => 'string',
 475         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 476         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 477         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 478         default => 'now',
 479     },
 480     startup =>  get_standard_option('pve-startup-order'),
 481     template => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Enable/disable Template.",
 485         default => 0,
 486     },
 487     args => {
 488         optional => 1,
 489         type => 'string',
 490         description => "Arbitrary arguments passed to kvm.",
 491         verbose_description => <<EODESCR,
 492 Arbitrary arguments passed to kvm, for example:
 493
 494 args: -no-reboot -no-hpet
 495
 496 NOTE: this option is for experts only.
 497 EODESCR
 498     },
 499     tablet => {
 500         optional => 1,
 501         type => 'boolean',
 502         default => 1,
 503         description => "Enable/disable the USB tablet device.",
 504         verbose_description => "Enable/disable the USB tablet device. This device is " .
 505             "usually needed to allow absolute mouse positioning with VNC. " .
 506             "Else the mouse runs out of sync with normal VNC clients. " .
 507             "If you're running lots of console-only guests on one host, " .
 508             "you may consider disabling this to save some context switches. " .
 509             "This is turned off by default if you use spice (-vga=qxl).",
 510     },
 511     migrate_speed => {
 512         optional => 1,
 513         type => 'integer',
 514         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 515         minimum => 0,
 516         default => 0,
 517     },
 518     migrate_downtime => {
 519         optional => 1,
 520         type => 'number',
 521         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 522         minimum => 0,
 523         default => 0.1,
 524     },
 525     cdrom => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-ide',
 528         typetext => '<volume>',
 529         description => "This is an alias for option -ide2",
 530     },
 531     cpu => {
 532         optional => 1,
 533         description => "Emulated CPU type.",
 534         type => 'string',
 535         format => $cpu_fmt,
 536     },
 537     parent => get_standard_option('pve-snapshot-name', {
 538         optional => 1,
 539         description => "Parent snapshot name. This is used internally, and should not be modified.",
 540     }),
 541     snaptime => {
 542         optional => 1,
 543         description => "Timestamp for snapshots.",
 544         type => 'integer',
 545         minimum => 0,
 546     },
 547     vmstate => {
 548         optional => 1,
 549         type => 'string', format => 'pve-volume-id',
 550         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 551     },
 552     vmstatestorage => get_standard_option('pve-storage-id', {
 553         description => "Default storage for VM state volumes/files.",
 554         optional => 1,
 555     }),
 556     runningmachine => get_standard_option('pve-qemu-machine', {
 557         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 558     }),
 559     machine => get_standard_option('pve-qemu-machine'),
 560     arch => {
 561         description => "Virtual processor architecture. Defaults to the host.",
 562         optional => 1,
 563         type => 'string',
 564         enum => [qw(x86_64 aarch64)],
 565     },
 566     smbios1 => {
 567         description => "Specify SMBIOS type 1 fields.",
 568         type => 'string', format => 'pve-qm-smbios1',
 569         maxLength => 256,
 570         optional => 1,
 571     },
 572     protection => {
 573         optional => 1,
 574         type => 'boolean',
 575         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 576         default => 0,
 577     },
 578     bios => {
 579         optional => 1,
 580         type => 'string',
 581         enum => [ qw(seabios ovmf) ],
 582         description => "Select BIOS implementation.",
 583         default => 'seabios',
 584     },
 585     vmgenid => {
 586         type => 'string',
 587         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 588         format_description => 'UUID',
 589         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 590         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 591             " 128-bit integer value identifier to the guest OS. This allows to".
 592             " notify the guest operating system when the virtual machine is".
 593             " executed with a different configuration (e.g. snapshot execution".
 594             " or creation from a template). The guest operating system notices".
 595             " the change, and is then able to react as appropriate by marking".
 596             " its copies of distributed databases as dirty, re-initializing its".
 597             " random number generator, etc.\n".
 598             "Note that auto-creation only works when done throug API/CLI create".
 599             " or update methods, but not when manually editing the config file.",
 600         default => "1 (autogenerated)",
 601         optional => 1,
 602     },
 603 };
 604
 605 my $confdesc_cloudinit = {
 606     citype => {
 607         optional => 1,
 608         type => 'string',
 609         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 610         enum => ['configdrive2', 'nocloud'],
 611     },
 612     ciuser => {
 613         optional => 1,
 614         type => 'string',
 615         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 616     },
 617     cipassword => {
 618         optional => 1,
 619         type => 'string',
 620         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 621     },
 622     searchdomain => {
 623         optional => 1,
 624         type => 'string',
 625         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 626     },
 627     nameserver => {
 628         optional => 1,
 629         type => 'string', format => 'address-list',
 630         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 631     },
 632     sshkeys => {
 633         optional => 1,
 634         type => 'string',
 635         format => 'urlencoded',
 636         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 637     },
 638 };
 639
 640 # what about other qemu settings ?
 641 #cpu => 'string',
 642 #machine => 'string',
 643 #fda => 'file',
 644 #fdb => 'file',
 645 #mtdblock => 'file',
 646 #sd => 'file',
 647 #pflash => 'file',
 648 #snapshot => 'bool',
 649 #bootp => 'file',
 650 ##tftp => 'dir',
 651 ##smb => 'dir',
 652 #kernel => 'file',
 653 #append => 'string',
 654 #initrd => 'file',
 655 ##soundhw => 'string',
 656
 657 while (my ($k, $v) = each %$confdesc) {
 658     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 659 }
 660
 661 my $MAX_IDE_DISKS = 4;
 662 my $MAX_SCSI_DISKS = 14;
 663 my $MAX_VIRTIO_DISKS = 16;
 664 my $MAX_SATA_DISKS = 6;
 665 my $MAX_USB_DEVICES = 5;
 666 my $MAX_NETS = 32;
 667 my $MAX_UNUSED_DISKS = 256;
 668 my $MAX_HOSTPCI_DEVICES = 4;
 669 my $MAX_SERIAL_PORTS = 4;
 670 my $MAX_PARALLEL_PORTS = 3;
 671 my $MAX_NUMA = 8;
 672
 673 my $numa_fmt = {
 674     cpus => {
 675         type => "string",
 676         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 677         description => "CPUs accessing this NUMA node.",
 678         format_description => "id[-id];...",
 679     },
 680     memory => {
 681         type => "number",
 682         description => "Amount of memory this NUMA node provides.",
 683         optional => 1,
 684     },
 685     hostnodes => {
 686         type => "string",
 687         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 688         description => "Host NUMA nodes to use.",
 689         format_description => "id[-id];...",
 690         optional => 1,
 691     },
 692     policy => {
 693         type => 'string',
 694         enum => [qw(preferred bind interleave)],
 695         description => "NUMA allocation policy.",
 696         optional => 1,
 697     },
 698 };
 699 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 700 my $numadesc = {
 701     optional => 1,
 702     type => 'string', format => $numa_fmt,
 703     description => "NUMA topology.",
 704 };
 705 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 706
 707 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 708     $confdesc->{"numa$i"} = $numadesc;
 709 }
 710
 711 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 712                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 713                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 714 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 715
 716 my $net_fmt_bridge_descr = <<__EOD__;
 717 Bridge to attach the network device to. The Proxmox VE standard bridge
 718 is called 'vmbr0'.
 719
 720 If you do not specify a bridge, we create a kvm user (NATed) network
 721 device, which provides DHCP and DNS services. The following addresses
 722 are used:
 723
 724  10.0.2.2   Gateway
 725  10.0.2.3   DNS Server
 726  10.0.2.4   SMB Server
 727
 728 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 729 __EOD__
 730
 731 my $net_fmt = {
 732     macaddr => {
 733         type => 'string',
 734         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 735         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 736         format_description => "XX:XX:XX:XX:XX:XX",
 737         optional => 1,
 738     },
 739     model => {
 740         type => 'string',
 741         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 742         enum => $nic_model_list,
 743         default_key => 1,
 744     },
 745     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 746     bridge => {
 747         type => 'string',
 748         description => $net_fmt_bridge_descr,
 749         format_description => 'bridge',
 750         optional => 1,
 751     },
 752     queues => {
 753         type => 'integer',
 754         minimum => 0, maximum => 16,
 755         description => 'Number of packet queues to be used on the device.',
 756         optional => 1,
 757     },
 758     rate => {
 759         type => 'number',
 760         minimum => 0,
 761         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 762         optional => 1,
 763     },
 764     tag => {
 765         type => 'integer',
 766         minimum => 1, maximum => 4094,
 767         description => 'VLAN tag to apply to packets on this interface.',
 768         optional => 1,
 769     },
 770     trunks => {
 771         type => 'string',
 772         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 773         description => 'VLAN trunks to pass through this interface.',
 774         format_description => 'vlanid[;vlanid...]',
 775         optional => 1,
 776     },
 777     firewall => {
 778         type => 'boolean',
 779         description => 'Whether this interface should be protected by the firewall.',
 780         optional => 1,
 781     },
 782     link_down => {
 783         type => 'boolean',
 784         description => 'Whether this interface should be disconnected (like pulling the plug).',
 785         optional => 1,
 786     },
 787 };
 788
 789 my $netdesc = {
 790     optional => 1,
 791     type => 'string', format => $net_fmt,
 792     description => "Specify network devices.",
 793 };
 794
 795 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 796
 797 my $ipconfig_fmt = {
 798     ip => {
 799         type => 'string',
 800         format => 'pve-ipv4-config',
 801         format_description => 'IPv4Format/CIDR',
 802         description => 'IPv4 address in CIDR format.',
 803         optional => 1,
 804         default => 'dhcp',
 805     },
 806     gw => {
 807         type => 'string',
 808         format => 'ipv4',
 809         format_description => 'GatewayIPv4',
 810         description => 'Default gateway for IPv4 traffic.',
 811         optional => 1,
 812         requires => 'ip',
 813     },
 814     ip6 => {
 815         type => 'string',
 816         format => 'pve-ipv6-config',
 817         format_description => 'IPv6Format/CIDR',
 818         description => 'IPv6 address in CIDR format.',
 819         optional => 1,
 820         default => 'dhcp',
 821     },
 822     gw6 => {
 823         type => 'string',
 824         format => 'ipv6',
 825         format_description => 'GatewayIPv6',
 826         description => 'Default gateway for IPv6 traffic.',
 827         optional => 1,
 828         requires => 'ip6',
 829     },
 830 };
 831 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 832 my $ipconfigdesc = {
 833     optional => 1,
 834     type => 'string', format => 'pve-qm-ipconfig',
 835     description => <<'EODESCR',
 836 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 837
 838 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 839
 840 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 841 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 842
 843 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 844 EODESCR
 845 };
 846 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 847
 848 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 849     $confdesc->{"net$i"} = $netdesc;
 850     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 851 }
 852
 853 foreach my $key (keys %$confdesc_cloudinit) {
 854     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 855 }
 856
 857 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 858 sub verify_volume_id_or_qm_path {
 859     my ($volid, $noerr) = @_;
 860
 861     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 862         return $volid;
 863     }
 864
 865     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 866     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 867     if ($@) {
 868         return undef if $noerr;
 869         die $@;
 870     }
 871     return $volid;
 872 }
 873
 874 my $drivename_hash;
 875
 876 my %drivedesc_base = (
 877     volume => { alias => 'file' },
 878     file => {
 879         type => 'string',
 880         format => 'pve-volume-id-or-qm-path',
 881         default_key => 1,
 882         format_description => 'volume',
 883         description => "The drive's backing volume.",
 884     },
 885     media => {
 886         type => 'string',
 887         enum => [qw(cdrom disk)],
 888         description => "The drive's media type.",
 889         default => 'disk',
 890         optional => 1
 891     },
 892     cyls => {
 893         type => 'integer',
 894         description => "Force the drive's physical geometry to have a specific cylinder count.",
 895         optional => 1
 896     },
 897     heads => {
 898         type => 'integer',
 899         description => "Force the drive's physical geometry to have a specific head count.",
 900         optional => 1
 901     },
 902     secs => {
 903         type => 'integer',
 904         description => "Force the drive's physical geometry to have a specific sector count.",
 905         optional => 1
 906     },
 907     trans => {
 908         type => 'string',
 909         enum => [qw(none lba auto)],
 910         description => "Force disk geometry bios translation mode.",
 911         optional => 1,
 912     },
 913     snapshot => {
 914         type => 'boolean',
 915         description => "Controls qemu's snapshot mode feature."
 916             . " If activated, changes made to the disk are temporary and will"
 917             . " be discarded when the VM is shutdown.",
 918         optional => 1,
 919     },
 920     cache => {
 921         type => 'string',
 922         enum => [qw(none writethrough writeback unsafe directsync)],
 923         description => "The drive's cache mode",
 924         optional => 1,
 925     },
 926     format => get_standard_option('pve-qm-image-format'),
 927     size => {
 928         type => 'string',
 929         format => 'disk-size',
 930         format_description => 'DiskSize',
 931         description => "Disk size. This is purely informational and has no effect.",
 932         optional => 1,
 933     },
 934     backup => {
 935         type => 'boolean',
 936         description => "Whether the drive should be included when making backups.",
 937         optional => 1,
 938     },
 939     replicate => {
 940         type => 'boolean',
 941         description => 'Whether the drive should considered for replication jobs.',
 942         optional => 1,
 943         default => 1,
 944     },
 945     rerror => {
 946         type => 'string',
 947         enum => [qw(ignore report stop)],
 948         description => 'Read error action.',
 949         optional => 1,
 950     },
 951     werror => {
 952         type => 'string',
 953         enum => [qw(enospc ignore report stop)],
 954         description => 'Write error action.',
 955         optional => 1,
 956     },
 957     aio => {
 958         type => 'string',
 959         enum => [qw(native threads)],
 960         description => 'AIO type to use.',
 961         optional => 1,
 962     },
 963     discard => {
 964         type => 'string',
 965         enum => [qw(ignore on)],
 966         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 967         optional => 1,
 968     },
 969     detect_zeroes => {
 970         type => 'boolean',
 971         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 972         optional => 1,
 973     },
 974     serial => {
 975         type => 'string',
 976         format => 'urlencoded',
 977         format_description => 'serial',
 978         maxLength => 20*3, # *3 since it's %xx url enoded
 979         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 980         optional => 1,
 981     },
 982     shared => {
 983         type => 'boolean',
 984         description => 'Mark this locally-managed volume as available on all nodes',
 985         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 986         optional => 1,
 987         default => 0,
 988     }
 989 );
 990
 991 my %iothread_fmt = ( iothread => {
 992         type => 'boolean',
 993         description => "Whether to use iothreads for this drive",
 994         optional => 1,
 995 });
 996
 997 my %model_fmt = (
 998     model => {
 999         type => 'string',
1000         format => 'urlencoded',
1001         format_description => 'model',
1002         maxLength => 40*3, # *3 since it's %xx url enoded
1003         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1004         optional => 1,
1005     },
1006 );
1007
1008 my %queues_fmt = (
1009     queues => {
1010         type => 'integer',
1011         description => "Number of queues.",
1012         minimum => 2,
1013         optional => 1
1014     }
1015 );
1016
1017 my %scsiblock_fmt = (
1018     scsiblock => {
1019         type => 'boolean',
1020         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1021         optional => 1,
1022         default => 0,
1023     },
1024 );
1025
1026 my %ssd_fmt = (
1027     ssd => {
1028         type => 'boolean',
1029         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1030         optional => 1,
1031     },
1032 );
1033
1034 my $add_throttle_desc = sub {
1035     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1036     my $d = {
1037         type => $type,
1038         format_description => $unit,
1039         description => "Maximum $what in $longunit.",
1040         optional => 1,
1041     };
1042     $d->{minimum} = $minimum if defined($minimum);
1043     $drivedesc_base{$key} = $d;
1044 };
1045 # throughput: (leaky bucket)
1046 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1047 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1048 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1049 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1050 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1051 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1052 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1053 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1054 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1055
1056 # pools: (pool of IO before throttling starts taking effect)
1057 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1059 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1060 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1061 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1062 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1063
1064 # burst lengths
1065 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1066 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1067 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1068 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1069 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1070 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1071
1072 # legacy support
1073 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1074 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1075 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1076 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1077
1078 my $ide_fmt = {
1079     %drivedesc_base,
1080     %model_fmt,
1081     %ssd_fmt,
1082 };
1083 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1084
1085 my $idedesc = {
1086     optional => 1,
1087     type => 'string', format => $ide_fmt,
1088     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1089 };
1090 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1091
1092 my $scsi_fmt = {
1093     %drivedesc_base,
1094     %iothread_fmt,
1095     %queues_fmt,
1096     %scsiblock_fmt,
1097     %ssd_fmt,
1098 };
1099 my $scsidesc = {
1100     optional => 1,
1101     type => 'string', format => $scsi_fmt,
1102     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1103 };
1104 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1105
1106 my $sata_fmt = {
1107     %drivedesc_base,
1108     %ssd_fmt,
1109 };
1110 my $satadesc = {
1111     optional => 1,
1112     type => 'string', format => $sata_fmt,
1113     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1114 };
1115 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1116
1117 my $virtio_fmt = {
1118     %drivedesc_base,
1119     %iothread_fmt,
1120 };
1121 my $virtiodesc = {
1122     optional => 1,
1123     type => 'string', format => $virtio_fmt,
1124     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1125 };
1126 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1127
1128 my $alldrive_fmt = {
1129     %drivedesc_base,
1130     %iothread_fmt,
1131     %model_fmt,
1132     %queues_fmt,
1133     %scsiblock_fmt,
1134     %ssd_fmt,
1135 };
1136
1137 my $efidisk_fmt = {
1138     volume => { alias => 'file' },
1139     file => {
1140         type => 'string',
1141         format => 'pve-volume-id-or-qm-path',
1142         default_key => 1,
1143         format_description => 'volume',
1144         description => "The drive's backing volume.",
1145     },
1146     format => get_standard_option('pve-qm-image-format'),
1147     size => {
1148         type => 'string',
1149         format => 'disk-size',
1150         format_description => 'DiskSize',
1151         description => "Disk size. This is purely informational and has no effect.",
1152         optional => 1,
1153     },
1154 };
1155
1156 my $efidisk_desc = {
1157     optional => 1,
1158     type => 'string', format => $efidisk_fmt,
1159     description => "Configure a Disk for storing EFI vars",
1160 };
1161
1162 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1163
1164 my $usb_fmt = {
1165     host => {
1166         default_key => 1,
1167         type => 'string', format => 'pve-qm-usb-device',
1168         format_description => 'HOSTUSBDEVICE|spice',
1169         description => <<EODESCR,
1170 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1171
1172  'bus-port(.port)*' (decimal numbers) or
1173  'vendor_id:product_id' (hexadeciaml numbers) or
1174  'spice'
1175
1176 You can use the 'lsusb -t' command to list existing usb devices.
1177
1178 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1179
1180 The value 'spice' can be used to add a usb redirection devices for spice.
1181 EODESCR
1182     },
1183     usb3 => {
1184         optional => 1,
1185         type => 'boolean',
1186         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1187         default => 0,
1188     },
1189 };
1190
1191 my $usbdesc = {
1192     optional => 1,
1193     type => 'string', format => $usb_fmt,
1194     description => "Configure an USB device (n is 0 to 4).",
1195 };
1196 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1197
1198 # NOTE: the match-groups of this regex are used in parse_hostpci
1199 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1200 my $hostpci_fmt = {
1201     host => {
1202         default_key => 1,
1203         type => 'string',
1204         pattern => qr/$PCIRE(;$PCIRE)*/,
1205         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1206         description => <<EODESCR,
1207 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1208 of PCI virtual functions of the host. HOSTPCIID syntax is:
1209
1210 'bus:dev.func' (hexadecimal numbers)
1211
1212 You can us the 'lspci' command to list existing PCI devices.
1213 EODESCR
1214     },
1215     rombar => {
1216         type => 'boolean',
1217         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1218         optional => 1,
1219         default => 1,
1220     },
1221     romfile => {
1222         type => 'string',
1223         pattern => '[^,;]+',
1224         format_description => 'string',
1225         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1226         optional => 1,
1227     },
1228     pcie => {
1229         type => 'boolean',
1230         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1231         optional => 1,
1232         default => 0,
1233     },
1234     'x-vga' => {
1235         type => 'boolean',
1236         description =>  "Enable vfio-vga device support.",
1237         optional => 1,
1238         default => 0,
1239     },
1240 };
1241 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1242
1243 my $hostpcidesc = {
1244         optional => 1,
1245         type => 'string', format => 'pve-qm-hostpci',
1246         description => "Map host PCI devices into guest.",
1247         verbose_description =>  <<EODESCR,
1248 Map host PCI devices into guest.
1249
1250 NOTE: This option allows direct access to host hardware. So it is no longer
1251 possible to migrate such machines - use with special care.
1252
1253 CAUTION: Experimental! User reported problems with this option.
1254 EODESCR
1255 };
1256 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1257
1258 my $serialdesc = {
1259         optional => 1,
1260         type => 'string',
1261         pattern => '(/dev/.+|socket)',
1262         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1263         verbose_description =>  <<EODESCR,
1264 Create a serial device inside the VM (n is 0 to 3), and pass through a
1265 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1266 host side (use 'qm terminal' to open a terminal connection).
1267
1268 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1269
1270 CAUTION: Experimental! User reported problems with this option.
1271 EODESCR
1272 };
1273
1274 my $paralleldesc= {
1275         optional => 1,
1276         type => 'string',
1277         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1278         description =>  "Map host parallel devices (n is 0 to 2).",
1279         verbose_description =>  <<EODESCR,
1280 Map host parallel devices (n is 0 to 2).
1281
1282 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1283
1284 CAUTION: Experimental! User reported problems with this option.
1285 EODESCR
1286 };
1287
1288 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1289     $confdesc->{"parallel$i"} = $paralleldesc;
1290 }
1291
1292 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1293     $confdesc->{"serial$i"} = $serialdesc;
1294 }
1295
1296 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1297     $confdesc->{"hostpci$i"} = $hostpcidesc;
1298 }
1299
1300 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1301     $drivename_hash->{"ide$i"} = 1;
1302     $confdesc->{"ide$i"} = $idedesc;
1303 }
1304
1305 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1306     $drivename_hash->{"sata$i"} = 1;
1307     $confdesc->{"sata$i"} = $satadesc;
1308 }
1309
1310 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1311     $drivename_hash->{"scsi$i"} = 1;
1312     $confdesc->{"scsi$i"} = $scsidesc ;
1313 }
1314
1315 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1316     $drivename_hash->{"virtio$i"} = 1;
1317     $confdesc->{"virtio$i"} = $virtiodesc;
1318 }
1319
1320 $drivename_hash->{efidisk0} = 1;
1321 $confdesc->{efidisk0} = $efidisk_desc;
1322
1323 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1324     $confdesc->{"usb$i"} = $usbdesc;
1325 }
1326
1327 my $unuseddesc = {
1328     optional => 1,
1329     type => 'string', format => 'pve-volume-id',
1330     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1331 };
1332
1333 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1334     $confdesc->{"unused$i"} = $unuseddesc;
1335 }
1336
1337 my $kvm_api_version = 0;
1338
1339 sub kvm_version {
1340     return $kvm_api_version if $kvm_api_version;
1341
1342     open my $fh, '<', '/dev/kvm'
1343         or return undef;
1344
1345     # 0xae00 => KVM_GET_API_VERSION
1346     $kvm_api_version = ioctl($fh, 0xae00, 0);
1347
1348     return $kvm_api_version;
1349 }
1350
1351 my $kvm_user_version;
1352
1353 sub kvm_user_version {
1354
1355     return $kvm_user_version if $kvm_user_version;
1356
1357     $kvm_user_version = 'unknown';
1358
1359     my $code = sub {
1360         my $line = shift;
1361         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1362             $kvm_user_version = $2;
1363         }
1364     };
1365
1366     eval { run_command("kvm -version", outfunc => $code); };
1367     warn $@ if $@;
1368
1369     return $kvm_user_version;
1370
1371 }
1372
1373 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1374
1375 sub valid_drive_names {
1376     # order is important - used to autoselect boot disk
1377     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1378             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1379             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1380             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1381             'efidisk0');
1382 }
1383
1384 sub is_valid_drivename {
1385     my $dev = shift;
1386
1387     return defined($drivename_hash->{$dev});
1388 }
1389
1390 sub option_exists {
1391     my $key = shift;
1392     return defined($confdesc->{$key});
1393 }
1394
1395 sub nic_models {
1396     return $nic_model_list;
1397 }
1398
1399 sub os_list_description {
1400
1401     return {
1402         other => 'Other',
1403         wxp => 'Windows XP',
1404         w2k => 'Windows 2000',
1405         w2k3 =>, 'Windows 2003',
1406         w2k8 => 'Windows 2008',
1407         wvista => 'Windows Vista',
1408         win7 => 'Windows 7',
1409         win8 => 'Windows 8/2012',
1410         win10 => 'Windows 10/2016',
1411         l24 => 'Linux 2.4',
1412         l26 => 'Linux 2.6',
1413     };
1414 }
1415
1416 my $cdrom_path;
1417
1418 sub get_cdrom_path {
1419
1420     return  $cdrom_path if $cdrom_path;
1421
1422     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1423     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1424     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1425 }
1426
1427 sub get_iso_path {
1428     my ($storecfg, $vmid, $cdrom) = @_;
1429
1430     if ($cdrom eq 'cdrom') {
1431         return get_cdrom_path();
1432     } elsif ($cdrom eq 'none') {
1433         return '';
1434     } elsif ($cdrom =~ m|^/|) {
1435         return $cdrom;
1436     } else {
1437         return PVE::Storage::path($storecfg, $cdrom);
1438     }
1439 }
1440
1441 # try to convert old style file names to volume IDs
1442 sub filename_to_volume_id {
1443     my ($vmid, $file, $media) = @_;
1444
1445      if (!($file eq 'none' || $file eq 'cdrom' ||
1446           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1447
1448         return undef if $file =~ m|/|;
1449
1450         if ($media && $media eq 'cdrom') {
1451             $file = "local:iso/$file";
1452         } else {
1453             $file = "local:$vmid/$file";
1454         }
1455     }
1456
1457     return $file;
1458 }
1459
1460 sub verify_media_type {
1461     my ($opt, $vtype, $media) = @_;
1462
1463     return if !$media;
1464
1465     my $etype;
1466     if ($media eq 'disk') {
1467         $etype = 'images';
1468     } elsif ($media eq 'cdrom') {
1469         $etype = 'iso';
1470     } else {
1471         die "internal error";
1472     }
1473
1474     return if ($vtype eq $etype);
1475
1476     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1477 }
1478
1479 sub cleanup_drive_path {
1480     my ($opt, $storecfg, $drive) = @_;
1481
1482     # try to convert filesystem paths to volume IDs
1483
1484     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1485         ($drive->{file} !~ m|^/dev/.+|) &&
1486         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1487         ($drive->{file} !~ m/^\d+$/)) {
1488         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1489         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1490         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1491         verify_media_type($opt, $vtype, $drive->{media});
1492         $drive->{file} = $volid;
1493     }
1494
1495     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1496 }
1497
1498 sub parse_hotplug_features {
1499     my ($data) = @_;
1500
1501     my $res = {};
1502
1503     return $res if $data eq '0';
1504
1505     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1506
1507     foreach my $feature (PVE::Tools::split_list($data)) {
1508         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1509             $res->{$1} = 1;
1510         } else {
1511             die "invalid hotplug feature '$feature'\n";
1512         }
1513     }
1514     return $res;
1515 }
1516
1517 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1518 sub pve_verify_hotplug_features {
1519     my ($value, $noerr) = @_;
1520
1521     return $value if parse_hotplug_features($value);
1522
1523     return undef if $noerr;
1524
1525     die "unable to parse hotplug option\n";
1526 }
1527
1528 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1529 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1530 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1531 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1532 #        [,iothread=on][,serial=serial][,model=model]
1533
1534 sub parse_drive {
1535     my ($key, $data) = @_;
1536
1537     my ($interface, $index);
1538
1539     if ($key =~ m/^([^\d]+)(\d+)$/) {
1540         $interface = $1;
1541         $index = $2;
1542     } else {
1543         return undef;
1544     }
1545
1546     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1547                                      : $confdesc->{$key}->{format};
1548     if (!$desc) {
1549         warn "invalid drive key: $key\n";
1550         return undef;
1551     }
1552     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1553     return undef if !$res;
1554     $res->{interface} = $interface;
1555     $res->{index} = $index;
1556
1557     my $error = 0;
1558     foreach my $opt (qw(bps bps_rd bps_wr)) {
1559         if (my $bps = defined(delete $res->{$opt})) {
1560             if (defined($res->{"m$opt"})) {
1561                 warn "both $opt and m$opt specified\n";
1562                 ++$error;
1563                 next;
1564             }
1565             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1566         }
1567     }
1568
1569     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1570     for my $requirement (
1571         [mbps_max => 'mbps'],
1572         [mbps_rd_max => 'mbps_rd'],
1573         [mbps_wr_max => 'mbps_wr'],
1574         [miops_max => 'miops'],
1575         [miops_rd_max => 'miops_rd'],
1576         [miops_wr_max => 'miops_wr'],
1577         [bps_max_length => 'mbps_max'],
1578         [bps_rd_max_length => 'mbps_rd_max'],
1579         [bps_wr_max_length => 'mbps_wr_max'],
1580         [iops_max_length => 'iops_max'],
1581         [iops_rd_max_length => 'iops_rd_max'],
1582         [iops_wr_max_length => 'iops_wr_max']) {
1583         my ($option, $requires) = @$requirement;
1584         if ($res->{$option} && !$res->{$requires}) {
1585             warn "$option requires $requires\n";
1586             ++$error;
1587         }
1588     }
1589
1590     return undef if $error;
1591
1592     return undef if $res->{mbps_rd} && $res->{mbps};
1593     return undef if $res->{mbps_wr} && $res->{mbps};
1594     return undef if $res->{iops_rd} && $res->{iops};
1595     return undef if $res->{iops_wr} && $res->{iops};
1596
1597     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1598         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1599         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1600         return undef if $res->{interface} eq 'virtio';
1601     }
1602
1603     if (my $size = $res->{size}) {
1604         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1605     }
1606
1607     return $res;
1608 }
1609
1610 sub print_drive {
1611     my ($vmid, $drive) = @_;
1612     my $data = { %$drive };
1613     delete $data->{$_} for qw(index interface);
1614     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1615 }
1616
1617 sub scsi_inquiry {
1618     my($fh, $noerr) = @_;
1619
1620     my $SG_IO = 0x2285;
1621     my $SG_GET_VERSION_NUM = 0x2282;
1622
1623     my $versionbuf = "\x00" x 8;
1624     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1625     if (!$ret) {
1626         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1627         return undef;
1628     }
1629     my $version = unpack("I", $versionbuf);
1630     if ($version < 30000) {
1631         die "scsi generic interface too old\n"  if !$noerr;
1632         return undef;
1633     }
1634
1635     my $buf = "\x00" x 36;
1636     my $sensebuf = "\x00" x 8;
1637     my $cmd = pack("C x3 C x1", 0x12, 36);
1638
1639     # see /usr/include/scsi/sg.h
1640     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1641
1642     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1643                       length($sensebuf), 0, length($buf), $buf,
1644                       $cmd, $sensebuf, 6000);
1645
1646     $ret = ioctl($fh, $SG_IO, $packet);
1647     if (!$ret) {
1648         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1649         return undef;
1650     }
1651
1652     my @res = unpack($sg_io_hdr_t, $packet);
1653     if ($res[17] || $res[18]) {
1654         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1655         return undef;
1656     }
1657
1658     my $res = {};
1659     (my $byte0, my $byte1, $res->{vendor},
1660      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1661
1662     $res->{removable} = $byte1 & 128 ? 1 : 0;
1663     $res->{type} = $byte0 & 31;
1664
1665     return $res;
1666 }
1667
1668 sub path_is_scsi {
1669     my ($path) = @_;
1670
1671     my $fh = IO::File->new("+<$path") || return undef;
1672     my $res = scsi_inquiry($fh, 1);
1673     close($fh);
1674
1675     return $res;
1676 }
1677
1678 sub machine_type_is_q35 {
1679     my ($conf) = @_;
1680
1681     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1682 }
1683
1684 sub print_tabletdevice_full {
1685     my ($conf) = @_;
1686
1687     my $q35 = machine_type_is_q35($conf);
1688
1689     # we use uhci for old VMs because tablet driver was buggy in older qemu
1690     my $usbbus = $q35 ? "ehci" : "uhci";
1691
1692     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1693 }
1694
1695 sub print_drivedevice_full {
1696     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1697
1698     my $device = '';
1699     my $maxdev = 0;
1700
1701     if ($drive->{interface} eq 'virtio') {
1702         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1703         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1704         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1705     } elsif ($drive->{interface} eq 'scsi') {
1706
1707         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1708         my $unit = $drive->{index} % $maxdev;
1709         my $devicetype = 'hd';
1710         my $path = '';
1711         if (drive_is_cdrom($drive)) {
1712             $devicetype = 'cd';
1713         } else {
1714             if ($drive->{file} =~ m|^/|) {
1715                 $path = $drive->{file};
1716                 if (my $info = path_is_scsi($path)) {
1717                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1718                         $devicetype = 'block';
1719                     } elsif ($info->{type} == 1) { # tape
1720                         $devicetype = 'generic';
1721                     }
1722                 }
1723             } else {
1724                  $path = PVE::Storage::path($storecfg, $drive->{file});
1725             }
1726
1727             if($path =~ m/^iscsi\:\/\//){
1728                 $devicetype = 'generic';
1729             }
1730         }
1731
1732         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1733             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1734         } else {
1735             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1736         }
1737
1738         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1739             $device .= ",rotation_rate=1";
1740         }
1741
1742     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1743         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1744         my $controller = int($drive->{index} / $maxdev);
1745         my $unit = $drive->{index} % $maxdev;
1746         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1747
1748         $device = "ide-$devicetype";
1749         if ($drive->{interface} eq 'ide') {
1750             $device .= ",bus=ide.$controller,unit=$unit";
1751         } else {
1752             $device .= ",bus=ahci$controller.$unit";
1753         }
1754         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1755
1756         if ($devicetype eq 'hd') {
1757             if (my $model = $drive->{model}) {
1758                 $model = URI::Escape::uri_unescape($model);
1759                 $device .= ",model=$model";
1760             }
1761             if ($drive->{ssd}) {
1762                 $device .= ",rotation_rate=1";
1763             }
1764         }
1765     } elsif ($drive->{interface} eq 'usb') {
1766         die "implement me";
1767         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1768     } else {
1769         die "unsupported interface type";
1770     }
1771
1772     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1773
1774     if (my $serial = $drive->{serial}) {
1775         $serial = URI::Escape::uri_unescape($serial);
1776         $device .= ",serial=$serial";
1777     }
1778
1779
1780     return $device;
1781 }
1782
1783 sub get_initiator_name {
1784     my $initiator;
1785
1786     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1787     while (defined(my $line = <$fh>)) {
1788         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1789         $initiator = $1;
1790         last;
1791     }
1792     $fh->close();
1793
1794     return $initiator;
1795 }
1796
1797 sub print_drive_full {
1798     my ($storecfg, $vmid, $drive) = @_;
1799
1800     my $path;
1801     my $volid = $drive->{file};
1802     my $format;
1803
1804     if (drive_is_cdrom($drive)) {
1805         $path = get_iso_path($storecfg, $vmid, $volid);
1806     } else {
1807         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1808         if ($storeid) {
1809             $path = PVE::Storage::path($storecfg, $volid);
1810             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1811             $format = qemu_img_format($scfg, $volname);
1812         } else {
1813             $path = $volid;
1814             $format = "raw";
1815         }
1816    }
1817
1818     my $opts = '';
1819     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1820     foreach my $o (@qemu_drive_options) {
1821         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1822     }
1823
1824     # snapshot only accepts on|off
1825     if (defined($drive->{snapshot})) {
1826         my $v = $drive->{snapshot} ? 'on' : 'off';
1827         $opts .= ",snapshot=$v";
1828     }
1829
1830     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1831         my ($dir, $qmpname) = @$type;
1832         if (my $v = $drive->{"mbps$dir"}) {
1833             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1834         }
1835         if (my $v = $drive->{"mbps${dir}_max"}) {
1836             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1837         }
1838         if (my $v = $drive->{"bps${dir}_max_length"}) {
1839             $opts .= ",throttling.bps$qmpname-max-length=$v";
1840         }
1841         if (my $v = $drive->{"iops${dir}"}) {
1842             $opts .= ",throttling.iops$qmpname=$v";
1843         }
1844         if (my $v = $drive->{"iops${dir}_max"}) {
1845             $opts .= ",throttling.iops$qmpname-max=$v";
1846         }
1847         if (my $v = $drive->{"iops${dir}_max_length"}) {
1848             $opts .= ",throttling.iops$qmpname-max-length=$v";
1849         }
1850     }
1851
1852     $opts .= ",format=$format" if $format && !$drive->{format};
1853
1854     my $cache_direct = 0;
1855
1856     if (my $cache = $drive->{cache}) {
1857         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1858     } elsif (!drive_is_cdrom($drive)) {
1859         $opts .= ",cache=none";
1860         $cache_direct = 1;
1861     }
1862
1863     # aio native works only with O_DIRECT
1864     if (!$drive->{aio}) {
1865         if($cache_direct) {
1866             $opts .= ",aio=native";
1867         } else {
1868             $opts .= ",aio=threads";
1869         }
1870     }
1871
1872     if (!drive_is_cdrom($drive)) {
1873         my $detectzeroes;
1874         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1875             $detectzeroes = 'off';
1876         } elsif ($drive->{discard}) {
1877             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1878         } else {
1879             # This used to be our default with discard not being specified:
1880             $detectzeroes = 'on';
1881         }
1882         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1883     }
1884
1885     my $pathinfo = $path ? "file=$path," : '';
1886
1887     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1888 }
1889
1890 sub print_netdevice_full {
1891     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1892
1893     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1894
1895     my $device = $net->{model};
1896     if ($net->{model} eq 'virtio') {
1897          $device = 'virtio-net-pci';
1898      };
1899
1900     my $pciaddr = print_pci_addr("$netid", $bridges);
1901     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1902     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1903         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1904         my $vectors = $net->{queues} * 2 + 2;
1905         $tmpstr .= ",vectors=$vectors,mq=on";
1906     }
1907     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1908
1909     if ($use_old_bios_files) {
1910         my $romfile;
1911         if ($device eq 'virtio-net-pci') {
1912             $romfile = 'pxe-virtio.rom';
1913         } elsif ($device eq 'e1000') {
1914             $romfile = 'pxe-e1000.rom';
1915         } elsif ($device eq 'ne2k') {
1916             $romfile = 'pxe-ne2k_pci.rom';
1917         } elsif ($device eq 'pcnet') {
1918             $romfile = 'pxe-pcnet.rom';
1919         } elsif ($device eq 'rtl8139') {
1920             $romfile = 'pxe-rtl8139.rom';
1921         }
1922         $tmpstr .= ",romfile=$romfile" if $romfile;
1923     }
1924
1925     return $tmpstr;
1926 }
1927
1928 sub print_netdev_full {
1929     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1930
1931     my $i = '';
1932     if ($netid =~ m/^net(\d+)$/) {
1933         $i = int($1);
1934     }
1935
1936     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1937
1938     my $ifname = "tap${vmid}i$i";
1939
1940     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1941     die "interface name '$ifname' is too long (max 15 character)\n"
1942         if length($ifname) >= 16;
1943
1944     my $vhostparam = '';
1945     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1946
1947     my $vmname = $conf->{name} || "vm$vmid";
1948
1949     my $netdev = "";
1950     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1951
1952     if ($net->{bridge}) {
1953         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1954     } else {
1955         $netdev = "type=user,id=$netid,hostname=$vmname";
1956     }
1957
1958     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1959
1960     return $netdev;
1961 }
1962
1963
1964 sub print_cpu_device {
1965     my ($conf, $id) = @_;
1966
1967     my $kvm = $conf->{kvm} // 1;
1968     my $cpu = $kvm ? "kvm64" : "qemu64";
1969     if (my $cputype = $conf->{cpu}) {
1970         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1971             or die "Cannot parse cpu description: $cputype\n";
1972         $cpu = $cpuconf->{cputype};
1973     }
1974
1975     my $cores = $conf->{cores} || 1;
1976
1977     my $current_core = ($id - 1) % $cores;
1978     my $current_socket = int(($id - 1 - $current_core)/$cores);
1979
1980     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1981 }
1982
1983 my $vga_map = {
1984     'cirrus' => 'cirrus-vga',
1985     'std' => 'VGA',
1986     'vmware' => 'vmware-svga',
1987     'virtio' => 'virtio-vga',
1988 };
1989
1990 sub print_vga_device {
1991     my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
1992
1993     my $type = $vga_map->{$vga->{type}};
1994     my $vgamem_mb = $vga->{memory};
1995     if ($qxlnum) {
1996         $type = $id ? 'qxl' : 'qxl-vga';
1997     }
1998     die "no devicetype for $vga->{type}\n" if !$type;
1999
2000     my $memory = "";
2001     if ($vgamem_mb) {
2002         if ($vga->{type} eq 'virtio') {
2003             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2004             $memory = ",max_hostmem=$bytes";
2005         } elsif ($qxlnum) {
2006             # from https://www.spice-space.org/multiple-monitors.html
2007             $memory = ",vgamem_mb=$vga->{memory}";
2008             my $ram = $vgamem_mb * 4;
2009             my $vram = $vgamem_mb * 2;
2010             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2011         } else {
2012             $memory = ",vgamem_mb=$vga->{memory}";
2013         }
2014     } elsif ($qxlnum && $id) {
2015         $memory = ",ram_size=67108864,vram_size=33554432";
2016     }
2017
2018     my $q35 = machine_type_is_q35($conf);
2019     my $vgaid = "vga" . ($id // '');
2020     my $pciaddr;
2021
2022     if ($q35 && $vgaid eq 'vga') {
2023         # the first display uses pcie.0 bus on q35 machines
2024         $pciaddr = print_pcie_addr($vgaid, $bridges);
2025     } else {
2026         $pciaddr = print_pci_addr($vgaid, $bridges);
2027     }
2028
2029     return "$type,id=${vgaid}${memory}${pciaddr}";
2030 }
2031
2032 sub drive_is_cloudinit {
2033     my ($drive) = @_;
2034     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2035 }
2036
2037 sub drive_is_cdrom {
2038     my ($drive, $exclude_cloudinit) = @_;
2039
2040     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2041
2042     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2043
2044 }
2045
2046 sub parse_number_sets {
2047     my ($set) = @_;
2048     my $res = [];
2049     foreach my $part (split(/;/, $set)) {
2050         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2051             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2052             push @$res, [ $1, $2 ];
2053         } else {
2054             die "invalid range: $part\n";
2055         }
2056     }
2057     return $res;
2058 }
2059
2060 sub parse_numa {
2061     my ($data) = @_;
2062
2063     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2064     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2065     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2066     return $res;
2067 }
2068
2069 sub parse_hostpci {
2070     my ($value) = @_;
2071
2072     return undef if !$value;
2073
2074     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2075
2076     my @idlist = split(/;/, $res->{host});
2077     delete $res->{host};
2078     foreach my $id (@idlist) {
2079         if ($id =~ /^$PCIRE$/) {
2080             if (defined($2)) {
2081                 push @{$res->{pciid}}, { id => $1, function => $2 };
2082             } else {
2083                 my $pcidevices = lspci($1);
2084                 $res->{pciid} = $pcidevices->{$1};
2085             }
2086         } else {
2087             # should have been caught by parse_property_string already
2088             die "failed to parse PCI id: $id\n";
2089         }
2090     }
2091     return $res;
2092 }
2093
2094 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2095 sub parse_net {
2096     my ($data) = @_;
2097
2098     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2099     if ($@) {
2100         warn $@;
2101         return undef;
2102     }
2103     if (!defined($res->{macaddr})) {
2104         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2105         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2106     }
2107     return $res;
2108 }
2109
2110 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2111 sub parse_ipconfig {
2112     my ($data) = @_;
2113
2114     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2115     if ($@) {
2116         warn $@;
2117         return undef;
2118     }
2119
2120     if ($res->{gw} && !$res->{ip}) {
2121         warn 'gateway specified without specifying an IP address';
2122         return undef;
2123     }
2124     if ($res->{gw6} && !$res->{ip6}) {
2125         warn 'IPv6 gateway specified without specifying an IPv6 address';
2126         return undef;
2127     }
2128     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2129         warn 'gateway specified together with DHCP';
2130         return undef;
2131     }
2132     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2133         # gw6 + auto/dhcp
2134         warn "IPv6 gateway specified together with $res->{ip6} address";
2135         return undef;
2136     }
2137
2138     if (!$res->{ip} && !$res->{ip6}) {
2139         return { ip => 'dhcp', ip6 => 'dhcp' };
2140     }
2141
2142     return $res;
2143 }
2144
2145 sub print_net {
2146     my $net = shift;
2147
2148     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2149 }
2150
2151 sub add_random_macs {
2152     my ($settings) = @_;
2153
2154     foreach my $opt (keys %$settings) {
2155         next if $opt !~ m/^net(\d+)$/;
2156         my $net = parse_net($settings->{$opt});
2157         next if !$net;
2158         $settings->{$opt} = print_net($net);
2159     }
2160 }
2161
2162 sub vm_is_volid_owner {
2163     my ($storecfg, $vmid, $volid) = @_;
2164
2165     if ($volid !~  m|^/|) {
2166         my ($path, $owner);
2167         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2168         if ($owner && ($owner == $vmid)) {
2169             return 1;
2170         }
2171     }
2172
2173     return undef;
2174 }
2175
2176 sub split_flagged_list {
2177     my $text = shift || '';
2178     $text =~ s/[,;]/ /g;
2179     $text =~ s/^\s+//;
2180     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2181 }
2182
2183 sub join_flagged_list {
2184     my ($how, $lst) = @_;
2185     join $how, map { $lst->{$_} . $_ } keys %$lst;
2186 }
2187
2188 sub vmconfig_delete_pending_option {
2189     my ($conf, $key, $force) = @_;
2190
2191     delete $conf->{pending}->{$key};
2192     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2193     $pending_delete_hash->{$key} = $force ? '!' : '';
2194     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2195 }
2196
2197 sub vmconfig_undelete_pending_option {
2198     my ($conf, $key) = @_;
2199
2200     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2201     delete $pending_delete_hash->{$key};
2202
2203     if (%$pending_delete_hash) {
2204         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2205     } else {
2206         delete $conf->{pending}->{delete};
2207     }
2208 }
2209
2210 sub vmconfig_register_unused_drive {
2211     my ($storecfg, $vmid, $conf, $drive) = @_;
2212
2213     if (drive_is_cloudinit($drive)) {
2214         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2215         warn $@ if $@;
2216     } elsif (!drive_is_cdrom($drive)) {
2217         my $volid = $drive->{file};
2218         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2219             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2220         }
2221     }
2222 }
2223
2224 sub vmconfig_cleanup_pending {
2225     my ($conf) = @_;
2226
2227     # remove pending changes when nothing changed
2228     my $changes;
2229     foreach my $opt (keys %{$conf->{pending}}) {
2230         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2231             $changes = 1;
2232             delete $conf->{pending}->{$opt};
2233         }
2234     }
2235
2236     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2237     my $pending_delete_hash = {};
2238     while (my ($opt, $force) = each %$current_delete_hash) {
2239         if (defined($conf->{$opt})) {
2240             $pending_delete_hash->{$opt} = $force;
2241         } else {
2242             $changes = 1;
2243         }
2244     }
2245
2246     if (%$pending_delete_hash) {
2247         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2248     } else {
2249         delete $conf->{pending}->{delete};
2250     }
2251
2252     return $changes;
2253 }
2254
2255 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2256 my $smbios1_fmt = {
2257     uuid => {
2258         type => 'string',
2259         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2260         format_description => 'UUID',
2261         description => "Set SMBIOS1 UUID.",
2262         optional => 1,
2263     },
2264     version => {
2265         type => 'string',
2266         pattern => '\S+',
2267         format_description => 'string',
2268         description => "Set SMBIOS1 version.",
2269         optional => 1,
2270     },
2271     serial => {
2272         type => 'string',
2273         pattern => '\S+',
2274         format_description => 'string',
2275         description => "Set SMBIOS1 serial number.",
2276         optional => 1,
2277     },
2278     manufacturer => {
2279         type => 'string',
2280         pattern => '\S+',
2281         format_description => 'string',
2282         description => "Set SMBIOS1 manufacturer.",
2283         optional => 1,
2284     },
2285     product => {
2286         type => 'string',
2287         pattern => '\S+',
2288         format_description => 'string',
2289         description => "Set SMBIOS1 product ID.",
2290         optional => 1,
2291     },
2292     sku => {
2293         type => 'string',
2294         pattern => '\S+',
2295         format_description => 'string',
2296         description => "Set SMBIOS1 SKU string.",
2297         optional => 1,
2298     },
2299     family => {
2300         type => 'string',
2301         pattern => '\S+',
2302         format_description => 'string',
2303         description => "Set SMBIOS1 family string.",
2304         optional => 1,
2305     },
2306 };
2307
2308 sub parse_smbios1 {
2309     my ($data) = @_;
2310
2311     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2312     warn $@ if $@;
2313     return $res;
2314 }
2315
2316 sub print_smbios1 {
2317     my ($smbios1) = @_;
2318     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2319 }
2320
2321 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2322
2323 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2324 sub verify_bootdisk {
2325     my ($value, $noerr) = @_;
2326
2327     return $value if is_valid_drivename($value);
2328
2329     return undef if $noerr;
2330
2331     die "invalid boot disk '$value'\n";
2332 }
2333
2334 sub parse_watchdog {
2335     my ($value) = @_;
2336
2337     return undef if !$value;
2338
2339     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2340     warn $@ if $@;
2341     return $res;
2342 }
2343
2344 sub parse_guest_agent {
2345     my ($value) = @_;
2346
2347     return {} if !defined($value->{agent});
2348
2349     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2350     warn $@ if $@;
2351
2352     # if the agent is disabled ignore the other potentially set properties
2353     return {} if !$res->{enabled};
2354     return $res;
2355 }
2356
2357 sub parse_vga {
2358     my ($value) = @_;
2359
2360     return {} if !$value;
2361     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2362     warn $@ if $@;
2363     return $res;
2364 }
2365
2366 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2367 sub verify_usb_device {
2368     my ($value, $noerr) = @_;
2369
2370     return $value if parse_usb_device($value);
2371
2372     return undef if $noerr;
2373
2374     die "unable to parse usb device\n";
2375 }
2376
2377 # add JSON properties for create and set function
2378 sub json_config_properties {
2379     my $prop = shift;
2380
2381     foreach my $opt (keys %$confdesc) {
2382         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2383         $prop->{$opt} = $confdesc->{$opt};
2384     }
2385
2386     return $prop;
2387 }
2388
2389 # return copy of $confdesc_cloudinit to generate documentation
2390 sub cloudinit_config_properties {
2391
2392     return dclone($confdesc_cloudinit);
2393 }
2394
2395 sub check_type {
2396     my ($key, $value) = @_;
2397
2398     die "unknown setting '$key'\n" if !$confdesc->{$key};
2399
2400     my $type = $confdesc->{$key}->{type};
2401
2402     if (!defined($value)) {
2403         die "got undefined value\n";
2404     }
2405
2406     if ($value =~ m/[\n\r]/) {
2407         die "property contains a line feed\n";
2408     }
2409
2410     if ($type eq 'boolean') {
2411         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2412         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2413         die "type check ('boolean') failed - got '$value'\n";
2414     } elsif ($type eq 'integer') {
2415         return int($1) if $value =~ m/^(\d+)$/;
2416         die "type check ('integer') failed - got '$value'\n";
2417     } elsif ($type eq 'number') {
2418         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2419         die "type check ('number') failed - got '$value'\n";
2420     } elsif ($type eq 'string') {
2421         if (my $fmt = $confdesc->{$key}->{format}) {
2422             PVE::JSONSchema::check_format($fmt, $value);
2423             return $value;
2424         }
2425         $value =~ s/^\"(.*)\"$/$1/;
2426         return $value;
2427     } else {
2428         die "internal error"
2429     }
2430 }
2431
2432 sub check_iommu_support{
2433     #fixme : need to check IOMMU support
2434     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2435
2436     my $iommu=1;
2437     return $iommu;
2438
2439 }
2440
2441 sub touch_config {
2442     my ($vmid) = @_;
2443
2444     my $conf = PVE::QemuConfig->config_file($vmid);
2445     utime undef, undef, $conf;
2446 }
2447
2448 sub destroy_vm {
2449     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2450
2451     my $conffile = PVE::QemuConfig->config_file($vmid);
2452
2453     my $conf = PVE::QemuConfig->load_config($vmid);
2454
2455     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2456
2457     if ($conf->{template}) {
2458         # check if any base image is still used by a linked clone
2459         foreach_drive($conf, sub {
2460                 my ($ds, $drive) = @_;
2461
2462                 return if drive_is_cdrom($drive);
2463
2464                 my $volid = $drive->{file};
2465
2466                 return if !$volid || $volid =~ m|^/|;
2467
2468                 die "base volume '$volid' is still in use by linked cloned\n"
2469                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2470
2471         });
2472     }
2473
2474     # only remove disks owned by this VM
2475     foreach_drive($conf, sub {
2476         my ($ds, $drive) = @_;
2477
2478         return if drive_is_cdrom($drive, 1);
2479
2480         my $volid = $drive->{file};
2481
2482         return if !$volid || $volid =~ m|^/|;
2483
2484         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2485         return if !$path || !$owner || ($owner != $vmid);
2486
2487         eval {
2488             PVE::Storage::vdisk_free($storecfg, $volid);
2489         };
2490         warn "Could not remove disk '$volid', check manually: $@" if $@;
2491
2492     });
2493
2494     if ($keep_empty_config) {
2495         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2496     } else {
2497         unlink $conffile;
2498     }
2499
2500     # also remove unused disk
2501     eval {
2502         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2503
2504         eval {
2505             PVE::Storage::foreach_volid($dl, sub {
2506                 my ($volid, $sid, $volname, $d) = @_;
2507                 PVE::Storage::vdisk_free($storecfg, $volid);
2508             });
2509         };
2510         warn $@ if $@;
2511
2512     };
2513     warn $@ if $@;
2514 }
2515
2516 sub parse_vm_config {
2517     my ($filename, $raw) = @_;
2518
2519     return undef if !defined($raw);
2520
2521     my $res = {
2522         digest => Digest::SHA::sha1_hex($raw),
2523         snapshots => {},
2524         pending => {},
2525     };
2526
2527     $filename =~ m|/qemu-server/(\d+)\.conf$|
2528         || die "got strange filename '$filename'";
2529
2530     my $vmid = $1;
2531
2532     my $conf = $res;
2533     my $descr;
2534     my $section = '';
2535
2536     my @lines = split(/\n/, $raw);
2537     foreach my $line (@lines) {
2538         next if $line =~ m/^\s*$/;
2539
2540         if ($line =~ m/^\[PENDING\]\s*$/i) {
2541             $section = 'pending';
2542             if (defined($descr)) {
2543                 $descr =~ s/\s+$//;
2544                 $conf->{description} = $descr;
2545             }
2546             $descr = undef;
2547             $conf = $res->{$section} = {};
2548             next;
2549
2550         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2551             $section = $1;
2552             if (defined($descr)) {
2553                 $descr =~ s/\s+$//;
2554                 $conf->{description} = $descr;
2555             }
2556             $descr = undef;
2557             $conf = $res->{snapshots}->{$section} = {};
2558             next;
2559         }
2560
2561         if ($line =~ m/^\#(.*)\s*$/) {
2562             $descr = '' if !defined($descr);
2563             $descr .= PVE::Tools::decode_text($1) . "\n";
2564             next;
2565         }
2566
2567         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2568             $descr = '' if !defined($descr);
2569             $descr .= PVE::Tools::decode_text($2);
2570         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2571             $conf->{snapstate} = $1;
2572         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2573             my $key = $1;
2574             my $value = $2;
2575             $conf->{$key} = $value;
2576         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2577             my $value = $1;
2578             if ($section eq 'pending') {
2579                 $conf->{delete} = $value; # we parse this later
2580             } else {
2581                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2582             }
2583         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2584             my $key = $1;
2585             my $value = $2;
2586             eval { $value = check_type($key, $value); };
2587             if ($@) {
2588                 warn "vm $vmid - unable to parse value of '$key' - $@";
2589             } else {
2590                 $key = 'ide2' if $key eq 'cdrom';
2591                 my $fmt = $confdesc->{$key}->{format};
2592                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2593                     my $v = parse_drive($key, $value);
2594                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2595                         $v->{file} = $volid;
2596                         $value = print_drive($vmid, $v);
2597                     } else {
2598                         warn "vm $vmid - unable to parse value of '$key'\n";
2599                         next;
2600                     }
2601                 }
2602
2603                 $conf->{$key} = $value;
2604             }
2605         }
2606     }
2607
2608     if (defined($descr)) {
2609         $descr =~ s/\s+$//;
2610         $conf->{description} = $descr;
2611     }
2612     delete $res->{snapstate}; # just to be sure
2613
2614     return $res;
2615 }
2616
2617 sub write_vm_config {
2618     my ($filename, $conf) = @_;
2619
2620     delete $conf->{snapstate}; # just to be sure
2621
2622     if ($conf->{cdrom}) {
2623         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2624         $conf->{ide2} = $conf->{cdrom};
2625         delete $conf->{cdrom};
2626     }
2627
2628     # we do not use 'smp' any longer
2629     if ($conf->{sockets}) {
2630         delete $conf->{smp};
2631     } elsif ($conf->{smp}) {
2632         $conf->{sockets} = $conf->{smp};
2633         delete $conf->{cores};
2634         delete $conf->{smp};
2635     }
2636
2637     my $used_volids = {};
2638
2639     my $cleanup_config = sub {
2640         my ($cref, $pending, $snapname) = @_;
2641
2642         foreach my $key (keys %$cref) {
2643             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2644                 $key eq 'snapstate' || $key eq 'pending';
2645             my $value = $cref->{$key};
2646             if ($key eq 'delete') {
2647                 die "propertry 'delete' is only allowed in [PENDING]\n"
2648                     if !$pending;
2649                 # fixme: check syntax?
2650                 next;
2651             }
2652             eval { $value = check_type($key, $value); };
2653             die "unable to parse value of '$key' - $@" if $@;
2654
2655             $cref->{$key} = $value;
2656
2657             if (!$snapname && is_valid_drivename($key)) {
2658                 my $drive = parse_drive($key, $value);
2659                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2660             }
2661         }
2662     };
2663
2664     &$cleanup_config($conf);
2665
2666     &$cleanup_config($conf->{pending}, 1);
2667
2668     foreach my $snapname (keys %{$conf->{snapshots}}) {
2669         die "internal error" if $snapname eq 'pending';
2670         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2671     }
2672
2673     # remove 'unusedX' settings if we re-add a volume
2674     foreach my $key (keys %$conf) {
2675         my $value = $conf->{$key};
2676         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2677             delete $conf->{$key};
2678         }
2679     }
2680
2681     my $generate_raw_config = sub {
2682         my ($conf, $pending) = @_;
2683
2684         my $raw = '';
2685
2686         # add description as comment to top of file
2687         if (defined(my $descr = $conf->{description})) {
2688             if ($descr) {
2689                 foreach my $cl (split(/\n/, $descr)) {
2690                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2691                 }
2692             } else {
2693                 $raw .= "#\n" if $pending;
2694             }
2695         }
2696
2697         foreach my $key (sort keys %$conf) {
2698             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2699             $raw .= "$key: $conf->{$key}\n";
2700         }
2701         return $raw;
2702     };
2703
2704     my $raw = &$generate_raw_config($conf);
2705
2706     if (scalar(keys %{$conf->{pending}})){
2707         $raw .= "\n[PENDING]\n";
2708         $raw .= &$generate_raw_config($conf->{pending}, 1);
2709     }
2710
2711     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2712         $raw .= "\n[$snapname]\n";
2713         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2714     }
2715
2716     return $raw;
2717 }
2718
2719 sub load_defaults {
2720
2721     my $res = {};
2722
2723     # we use static defaults from our JSON schema configuration
2724     foreach my $key (keys %$confdesc) {
2725         if (defined(my $default = $confdesc->{$key}->{default})) {
2726             $res->{$key} = $default;
2727         }
2728     }
2729
2730     return $res;
2731 }
2732
2733 sub config_list {
2734     my $vmlist = PVE::Cluster::get_vmlist();
2735     my $res = {};
2736     return $res if !$vmlist || !$vmlist->{ids};
2737     my $ids = $vmlist->{ids};
2738
2739     foreach my $vmid (keys %$ids) {
2740         my $d = $ids->{$vmid};
2741         next if !$d->{node} || $d->{node} ne $nodename;
2742         next if !$d->{type} || $d->{type} ne 'qemu';
2743         $res->{$vmid}->{exists} = 1;
2744     }
2745     return $res;
2746 }
2747
2748 # test if VM uses local resources (to prevent migration)
2749 sub check_local_resources {
2750     my ($conf, $noerr) = @_;
2751
2752     my $loc_res = 0;
2753
2754     $loc_res = 1 if $conf->{hostusb}; # old syntax
2755     $loc_res = 1 if $conf->{hostpci}; # old syntax
2756
2757     foreach my $k (keys %$conf) {
2758         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2759         # sockets are safe: they will recreated be on the target side post-migrate
2760         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2761         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2762     }
2763
2764     die "VM uses local resources\n" if $loc_res && !$noerr;
2765
2766     return $loc_res;
2767 }
2768
2769 # check if used storages are available on all nodes (use by migrate)
2770 sub check_storage_availability {
2771     my ($storecfg, $conf, $node) = @_;
2772
2773     foreach_drive($conf, sub {
2774         my ($ds, $drive) = @_;
2775
2776         my $volid = $drive->{file};
2777         return if !$volid;
2778
2779         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2780         return if !$sid;
2781
2782         # check if storage is available on both nodes
2783         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2784         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2785    });
2786 }
2787
2788 # list nodes where all VM images are available (used by has_feature API)
2789 sub shared_nodes {
2790     my ($conf, $storecfg) = @_;
2791
2792     my $nodelist = PVE::Cluster::get_nodelist();
2793     my $nodehash = { map { $_ => 1 } @$nodelist };
2794     my $nodename = PVE::INotify::nodename();
2795
2796     foreach_drive($conf, sub {
2797         my ($ds, $drive) = @_;
2798
2799         my $volid = $drive->{file};
2800         return if !$volid;
2801
2802         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2803         if ($storeid) {
2804             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2805             if ($scfg->{disable}) {
2806                 $nodehash = {};
2807             } elsif (my $avail = $scfg->{nodes}) {
2808                 foreach my $node (keys %$nodehash) {
2809                     delete $nodehash->{$node} if !$avail->{$node};
2810                 }
2811             } elsif (!$scfg->{shared}) {
2812                 foreach my $node (keys %$nodehash) {
2813                     delete $nodehash->{$node} if $node ne $nodename
2814                 }
2815             }
2816         }
2817     });
2818
2819     return $nodehash
2820 }
2821
2822 sub check_cmdline {
2823     my ($pidfile, $pid) = @_;
2824
2825     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2826     if (defined($fh)) {
2827         my $line = <$fh>;
2828         $fh->close;
2829         return undef if !$line;
2830         my @param = split(/\0/, $line);
2831
2832         my $cmd = $param[0];
2833         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2834
2835         for (my $i = 0; $i < scalar (@param); $i++) {
2836             my $p = $param[$i];
2837             next if !$p;
2838             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2839                 my $p = $param[$i+1];
2840                 return 1 if $p && ($p eq $pidfile);
2841                 return undef;
2842             }
2843         }
2844     }
2845     return undef;
2846 }
2847
2848 sub check_running {
2849     my ($vmid, $nocheck, $node) = @_;
2850
2851     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2852
2853     die "unable to find configuration file for VM $vmid - no such machine\n"
2854         if !$nocheck && ! -f $filename;
2855
2856     my $pidfile = pidfile_name($vmid);
2857
2858     if (my $fd = IO::File->new("<$pidfile")) {
2859         my $st = stat($fd);
2860         my $line = <$fd>;
2861         close($fd);
2862
2863         my $mtime = $st->mtime;
2864         if ($mtime > time()) {
2865             warn "file '$filename' modified in future\n";
2866         }
2867
2868         if ($line =~ m/^(\d+)$/) {
2869             my $pid = $1;
2870             if (check_cmdline($pidfile, $pid)) {
2871                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2872                     return $pid;
2873                 }
2874             }
2875         }
2876     }
2877
2878     return undef;
2879 }
2880
2881 sub vzlist {
2882
2883     my $vzlist = config_list();
2884
2885     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2886
2887     while (defined(my $de = $fd->read)) {
2888         next if $de !~ m/^(\d+)\.pid$/;
2889         my $vmid = $1;
2890         next if !defined($vzlist->{$vmid});
2891         if (my $pid = check_running($vmid)) {
2892             $vzlist->{$vmid}->{pid} = $pid;
2893         }
2894     }
2895
2896     return $vzlist;
2897 }
2898
2899 sub disksize {
2900     my ($storecfg, $conf) = @_;
2901
2902     my $bootdisk = $conf->{bootdisk};
2903     return undef if !$bootdisk;
2904     return undef if !is_valid_drivename($bootdisk);
2905
2906     return undef if !$conf->{$bootdisk};
2907
2908     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2909     return undef if !defined($drive);
2910
2911     return undef if drive_is_cdrom($drive);
2912
2913     my $volid = $drive->{file};
2914     return undef if !$volid;
2915
2916     return $drive->{size};
2917 }
2918
2919 our $vmstatus_return_properties = {
2920     vmid => get_standard_option('pve-vmid'),
2921     status => {
2922         description => "Qemu process status.",
2923         type => 'string',
2924         enum => ['stopped', 'running'],
2925     },
2926     maxmem => {
2927         description => "Maximum memory in bytes.",
2928         type => 'integer',
2929         optional => 1,
2930         renderer => 'bytes',
2931     },
2932     maxdisk => {
2933         description => "Root disk size in bytes.",
2934         type => 'integer',
2935         optional => 1,
2936         renderer => 'bytes',
2937     },
2938     name => {
2939         description => "VM name.",
2940         type => 'string',
2941         optional => 1,
2942     },
2943     qmpstatus => {
2944         description => "Qemu QMP agent status.",
2945         type => 'string',
2946         optional => 1,
2947     },
2948     pid => {
2949         description => "PID of running qemu process.",
2950         type => 'integer',
2951         optional => 1,
2952     },
2953     uptime => {
2954         description => "Uptime.",
2955         type => 'integer',
2956         optional => 1,
2957         renderer => 'duration',
2958     },
2959     cpus => {
2960         description => "Maximum usable CPUs.",
2961         type => 'number',
2962         optional => 1,
2963     },
2964 };
2965
2966 my $last_proc_pid_stat;
2967
2968 # get VM status information
2969 # This must be fast and should not block ($full == false)
2970 # We only query KVM using QMP if $full == true (this can be slow)
2971 sub vmstatus {
2972     my ($opt_vmid, $full) = @_;
2973
2974     my $res = {};
2975
2976     my $storecfg = PVE::Storage::config();
2977
2978     my $list = vzlist();
2979     my $defaults = load_defaults();
2980
2981     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2982
2983     my $cpucount = $cpuinfo->{cpus} || 1;
2984
2985     foreach my $vmid (keys %$list) {
2986         next if $opt_vmid && ($vmid ne $opt_vmid);
2987
2988         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2989         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2990
2991         my $d = { vmid => $vmid };
2992         $d->{pid} = $list->{$vmid}->{pid};
2993
2994         # fixme: better status?
2995         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2996
2997         my $size = disksize($storecfg, $conf);
2998         if (defined($size)) {
2999             $d->{disk} = 0; # no info available
3000             $d->{maxdisk} = $size;
3001         } else {
3002             $d->{disk} = 0;
3003             $d->{maxdisk} = 0;
3004         }
3005
3006         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3007             * ($conf->{cores} || $defaults->{cores});
3008         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3009         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3010
3011         $d->{name} = $conf->{name} || "VM $vmid";
3012         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3013             : $defaults->{memory}*(1024*1024);
3014
3015         if ($conf->{balloon}) {
3016             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3017             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3018                 : $defaults->{shares};
3019         }
3020
3021         $d->{uptime} = 0;
3022         $d->{cpu} = 0;
3023         $d->{mem} = 0;
3024
3025         $d->{netout} = 0;
3026         $d->{netin} = 0;
3027
3028         $d->{diskread} = 0;
3029         $d->{diskwrite} = 0;
3030
3031         $d->{template} = PVE::QemuConfig->is_template($conf);
3032
3033         $d->{serial} = 1 if conf_has_serial($conf);
3034
3035         $res->{$vmid} = $d;
3036     }
3037
3038     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3039     foreach my $dev (keys %$netdev) {
3040         next if $dev !~ m/^tap([1-9]\d*)i/;
3041         my $vmid = $1;
3042         my $d = $res->{$vmid};
3043         next if !$d;
3044
3045         $d->{netout} += $netdev->{$dev}->{receive};
3046         $d->{netin} += $netdev->{$dev}->{transmit};
3047
3048         if ($full) {
3049             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3050             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3051         }
3052
3053     }
3054
3055     my $ctime = gettimeofday;
3056
3057     foreach my $vmid (keys %$list) {
3058
3059         my $d = $res->{$vmid};
3060         my $pid = $d->{pid};
3061         next if !$pid;
3062
3063         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3064         next if !$pstat; # not running
3065
3066         my $used = $pstat->{utime} + $pstat->{stime};
3067
3068         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3069
3070         if ($pstat->{vsize}) {
3071             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3072         }
3073
3074         my $old = $last_proc_pid_stat->{$pid};
3075         if (!$old) {
3076             $last_proc_pid_stat->{$pid} = {
3077                 time => $ctime,
3078                 used => $used,
3079                 cpu => 0,
3080             };
3081             next;
3082         }
3083
3084         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3085
3086         if ($dtime > 1000) {
3087             my $dutime = $used -  $old->{used};
3088
3089             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3090             $last_proc_pid_stat->{$pid} = {
3091                 time => $ctime,
3092                 used => $used,
3093                 cpu => $d->{cpu},
3094             };
3095         } else {
3096             $d->{cpu} = $old->{cpu};
3097         }
3098     }
3099
3100     return $res if !$full;
3101
3102     my $qmpclient = PVE::QMPClient->new();
3103
3104     my $ballooncb = sub {
3105         my ($vmid, $resp) = @_;
3106
3107         my $info = $resp->{'return'};
3108         return if !$info->{max_mem};
3109
3110         my $d = $res->{$vmid};
3111
3112         # use memory assigned to VM
3113         $d->{maxmem} = $info->{max_mem};
3114         $d->{balloon} = $info->{actual};
3115
3116         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3117             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3118             $d->{freemem} = $info->{free_mem};
3119         }
3120
3121         $d->{ballooninfo} = $info;
3122     };
3123
3124     my $blockstatscb = sub {
3125         my ($vmid, $resp) = @_;
3126         my $data = $resp->{'return'} || [];
3127         my $totalrdbytes = 0;
3128         my $totalwrbytes = 0;
3129
3130         for my $blockstat (@$data) {
3131             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3132             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3133
3134             $blockstat->{device} =~ s/drive-//;
3135             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3136         }
3137         $res->{$vmid}->{diskread} = $totalrdbytes;
3138         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3139     };
3140
3141     my $statuscb = sub {
3142         my ($vmid, $resp) = @_;
3143
3144         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3145         # this fails if ballon driver is not loaded, so this must be
3146         # the last commnand (following command are aborted if this fails).
3147         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3148
3149         my $status = 'unknown';
3150         if (!defined($status = $resp->{'return'}->{status})) {
3151             warn "unable to get VM status\n";
3152             return;
3153         }
3154
3155         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3156     };
3157
3158     foreach my $vmid (keys %$list) {
3159         next if $opt_vmid && ($vmid ne $opt_vmid);
3160         next if !$res->{$vmid}->{pid}; # not running
3161         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3162     }
3163
3164     $qmpclient->queue_execute(undef, 2);
3165
3166     foreach my $vmid (keys %$list) {
3167         next if $opt_vmid && ($vmid ne $opt_vmid);
3168         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3169     }
3170
3171     return $res;
3172 }
3173
3174 sub foreach_drive {
3175     my ($conf, $func, @param) = @_;
3176
3177     foreach my $ds (valid_drive_names()) {
3178         next if !defined($conf->{$ds});
3179
3180         my $drive = parse_drive($ds, $conf->{$ds});
3181         next if !$drive;
3182
3183         &$func($ds, $drive, @param);
3184     }
3185 }
3186
3187 sub foreach_volid {
3188     my ($conf, $func, @param) = @_;
3189
3190     my $volhash = {};
3191
3192     my $test_volid = sub {
3193         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3194
3195         return if !$volid;
3196
3197         $volhash->{$volid}->{cdrom} //= 1;
3198         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3199
3200         $volhash->{$volid}->{replicate} //= 0;
3201         $volhash->{$volid}->{replicate} = 1 if $replicate;
3202
3203         $volhash->{$volid}->{shared} //= 0;
3204         $volhash->{$volid}->{shared} = 1 if $shared;
3205
3206         $volhash->{$volid}->{referenced_in_config} //= 0;
3207         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3208
3209         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3210             if defined($snapname);
3211     };
3212
3213     foreach_drive($conf, sub {
3214         my ($ds, $drive) = @_;
3215         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3216     });
3217
3218     foreach my $snapname (keys %{$conf->{snapshots}}) {
3219         my $snap = $conf->{snapshots}->{$snapname};
3220         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3221         foreach_drive($snap, sub {
3222             my ($ds, $drive) = @_;
3223             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3224         });
3225     }
3226
3227     foreach my $volid (keys %$volhash) {
3228         &$func($volid, $volhash->{$volid}, @param);
3229     }
3230 }
3231
3232 sub conf_has_serial {
3233     my ($conf) = @_;
3234
3235     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3236         if ($conf->{"serial$i"}) {
3237             return 1;
3238         }
3239     }
3240
3241     return 0;
3242 }
3243
3244 sub vga_conf_has_spice {
3245     my ($vga) = @_;
3246
3247     my $vgaconf = parse_vga($vga);
3248     my $vgatype = $vgaconf->{type};
3249     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3250
3251     return $1 || 1;
3252 }
3253
3254 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3255 sub get_host_arch() {
3256     $host_arch = (POSIX::uname())[4] if !$host_arch;
3257     return $host_arch;
3258 }
3259
3260 sub is_native($) {
3261     my ($arch) = @_;
3262     return get_host_arch() eq $arch;
3263 }
3264
3265 my $default_machines = {
3266     x86_64 => 'pc',
3267     aarch64 => 'virt',
3268 };
3269
3270 sub get_basic_machine_info {
3271     my ($conf, $forcemachine) = @_;
3272
3273     my $arch = $conf->{arch} // get_host_arch();
3274     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3275     return ($arch, $machine);
3276 }
3277
3278 sub config_to_command {
3279     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3280
3281     my $cmd = [];
3282     my $globalFlags = [];
3283     my $machineFlags = [];
3284     my $rtcFlags = [];
3285     my $cpuFlags = [];
3286     my $devices = [];
3287     my $pciaddr = '';
3288     my $bridges = {};
3289     my $kvmver = kvm_user_version();
3290     my $vernum = 0; # unknown
3291     my $ostype = $conf->{ostype};
3292     my $winversion = windows_version($ostype);
3293     my $kvm = $conf->{kvm};
3294
3295     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3296     $kvm //= 1 if is_native($arch);
3297
3298     if ($kvm) {
3299         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3300             if !defined kvm_version();
3301     }
3302
3303     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3304         $vernum = $1*1000000+$2*1000;
3305     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3306         $vernum = $1*1000000+$2*1000+$3;
3307     }
3308
3309     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3310
3311     my $have_ovz = -f '/proc/vz/vestat';
3312
3313     my $q35 = machine_type_is_q35($conf);
3314     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3315     my $use_old_bios_files = undef;
3316     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3317
3318     my $cpuunits = defined($conf->{cpuunits}) ?
3319             $conf->{cpuunits} : $defaults->{cpuunits};
3320
3321     push @$cmd, '/usr/bin/kvm';
3322
3323     push @$cmd, '-id', $vmid;
3324
3325     my $vmname = $conf->{name} || "vm$vmid";
3326
3327     push @$cmd, '-name', $vmname;
3328
3329     my $use_virtio = 0;
3330
3331     my $qmpsocket = qmp_socket($vmid);
3332     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3333     push @$cmd, '-mon', "chardev=qmp,mode=control";
3334
3335     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3336         my $eventsocket = qmp_socket($vmid, 0, 'event');
3337         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3338         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3339     }
3340
3341     push @$cmd, '-pidfile' , pidfile_name($vmid);
3342
3343     push @$cmd, '-daemonize';
3344
3345     if ($conf->{smbios1}) {
3346         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3347     }
3348
3349     if ($conf->{vmgenid}) {
3350         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3351     }
3352
3353     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3354         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3355
3356         my $path;
3357         my $format;
3358         if (my $efidisk = $conf->{efidisk0}) {
3359             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3360             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3361             $format = $d->{format};
3362             if ($storeid) {
3363                 $path = PVE::Storage::path($storecfg, $d->{file});
3364                 if (!defined($format)) {
3365                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3366                     $format = qemu_img_format($scfg, $volname);
3367                 }
3368             } else {
3369                 $path = $d->{file};
3370                 die "efidisk format must be specified\n"
3371                     if !defined($format);
3372             }
3373         } else {
3374             warn "no efidisk configured! Using temporary efivars disk.\n";
3375             $path = "/tmp/$vmid-ovmf.fd";
3376             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3377             $format = 'raw';
3378         }
3379
3380         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3381         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3382     }
3383
3384
3385     # add usb controllers
3386     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3387     push @$devices, @usbcontrollers if @usbcontrollers;
3388     my $vga = parse_vga($conf->{vga});
3389
3390     my $qxlnum = vga_conf_has_spice($conf->{vga});
3391     $vga->{type} = 'qxl' if $qxlnum;
3392
3393     if (!$vga->{type}) {
3394         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3395             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3396         } else {
3397             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3398         }
3399     }
3400
3401     # enable absolute mouse coordinates (needed by vnc)
3402     my $tablet;
3403     if (defined($conf->{tablet})) {
3404         $tablet = $conf->{tablet};
3405     } else {
3406         $tablet = $defaults->{tablet};
3407         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3408         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3409     }
3410
3411     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3412
3413     my $kvm_off = 0;
3414     my $gpu_passthrough;
3415
3416     # host pci devices
3417     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3418         my $d = parse_hostpci($conf->{"hostpci$i"});
3419         next if !$d;
3420
3421         my $pcie = $d->{pcie};
3422         if($pcie){
3423             die "q35 machine model is not enabled" if !$q35;
3424             $pciaddr = print_pcie_addr("hostpci$i");
3425         }else{
3426             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3427         }
3428
3429         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3430         my $romfile = $d->{romfile};
3431
3432         my $xvga = '';
3433         if ($d->{'x-vga'}) {
3434             $xvga = ',x-vga=on';
3435             $kvm_off = 1;
3436             $vga->{type} = 'none';
3437             $gpu_passthrough = 1;
3438
3439             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3440                 $xvga = "";
3441             }
3442         }
3443         my $pcidevices = $d->{pciid};
3444         my $multifunction = 1 if @$pcidevices > 1;
3445
3446         my $j=0;
3447         foreach my $pcidevice (@$pcidevices) {
3448
3449             my $id = "hostpci$i";
3450             $id .= ".$j" if $multifunction;
3451             my $addr = $pciaddr;
3452             $addr .= ".$j" if $multifunction;
3453             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3454
3455             if($j == 0){
3456                 $devicestr .= "$rombar$xvga";
3457                 $devicestr .= ",multifunction=on" if $multifunction;
3458                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3459             }
3460
3461             push @$devices, '-device', $devicestr;
3462             $j++;
3463         }
3464     }
3465
3466     # usb devices
3467     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3468     push @$devices, @usbdevices if @usbdevices;
3469     # serial devices
3470     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3471         if (my $path = $conf->{"serial$i"}) {
3472             if ($path eq 'socket') {
3473                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3474                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3475                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3476             } else {
3477                 die "no such serial device\n" if ! -c $path;
3478                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3479                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3480             }
3481         }
3482     }
3483
3484     # parallel devices
3485     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3486         if (my $path = $conf->{"parallel$i"}) {
3487             die "no such parallel device\n" if ! -c $path;
3488             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3489             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3490             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3491         }
3492     }
3493
3494
3495     my $sockets = 1;
3496     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3497     $sockets = $conf->{sockets} if  $conf->{sockets};
3498
3499     my $cores = $conf->{cores} || 1;
3500
3501     my $maxcpus = $sockets * $cores;
3502
3503     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3504
3505     my $allowed_vcpus = $cpuinfo->{cpus};
3506
3507     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3508         if ($allowed_vcpus < $maxcpus);
3509
3510     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3511
3512         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3513         for (my $i = 2; $i <= $vcpus; $i++)  {
3514             my $cpustr = print_cpu_device($conf,$i);
3515             push @$cmd, '-device', $cpustr;
3516         }
3517
3518     } else {
3519
3520         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3521     }
3522     push @$cmd, '-nodefaults';
3523
3524     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3525
3526     my $bootindex_hash = {};
3527     my $i = 1;
3528     foreach my $o (split(//, $bootorder)) {
3529         $bootindex_hash->{$o} = $i*100;
3530         $i++;
3531     }
3532
3533     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3534
3535     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3536
3537     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3538
3539     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3540         push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
3541         my $socket = vnc_socket($vmid);
3542         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3543     } else {
3544         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3545         push @$cmd, '-nographic';
3546     }
3547
3548     # time drift fix
3549     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3550
3551     my $useLocaltime = $conf->{localtime};
3552
3553     if ($winversion >= 5) { # windows
3554         $useLocaltime = 1 if !defined($conf->{localtime});
3555
3556         # use time drift fix when acpi is enabled
3557         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3558             $tdf = 1 if !defined($conf->{tdf});
3559         }
3560     }
3561
3562     if ($winversion >= 6) {
3563         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3564         push @$cmd, '-no-hpet';
3565     }
3566
3567     push @$rtcFlags, 'driftfix=slew' if $tdf;
3568
3569     if (!$kvm) {
3570         push @$machineFlags, 'accel=tcg';
3571     }
3572
3573     if ($machine_type) {
3574         push @$machineFlags, "type=${machine_type}";
3575     }
3576
3577     if ($conf->{startdate}) {
3578         push @$rtcFlags, "base=$conf->{startdate}";
3579     } elsif ($useLocaltime) {
3580         push @$rtcFlags, 'base=localtime';
3581     }
3582
3583     my $cpu = $kvm ? "kvm64" : "qemu64";
3584     if (my $cputype = $conf->{cpu}) {
3585         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3586             or die "Cannot parse cpu description: $cputype\n";
3587         $cpu = $cpuconf->{cputype};
3588         $kvm_off = 1 if $cpuconf->{hidden};
3589
3590         if (defined(my $flags = $cpuconf->{flags})) {
3591             push @$cpuFlags, split(";", $flags);
3592         }
3593     }
3594
3595     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3596
3597     push @$cpuFlags , '-x2apic'
3598         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3599
3600     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3601
3602     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3603
3604     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3605
3606         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3607         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3608     }
3609
3610     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3611
3612     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3613
3614     push @$cpuFlags, 'kvm=off' if $kvm_off;
3615
3616     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3617         die "internal error"; # should not happen
3618
3619     push @$cpuFlags, "vendor=${cpu_vendor}"
3620         if $cpu_vendor ne 'default';
3621
3622     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3623
3624     push @$cmd, '-cpu', $cpu;
3625
3626     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3627
3628     push @$cmd, '-S' if $conf->{freeze};
3629
3630     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3631
3632     # enable sound
3633     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3634     #push @$cmd, '-soundhw', 'es1370';
3635     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3636
3637     if (parse_guest_agent($conf)->{enabled}) {
3638         my $qgasocket = qmp_socket($vmid, 1);
3639         my $pciaddr = print_pci_addr("qga0", $bridges);
3640         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3641         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3642         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3643     }
3644
3645     my $spice_port;
3646
3647     if ($qxlnum) {
3648         if ($qxlnum > 1) {
3649             if ($winversion){
3650                 for(my $i = 1; $i < $qxlnum; $i++){
3651                     push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
3652                 }
3653             } else {
3654                 # assume other OS works like Linux
3655                 my ($ram, $vram) = ("134217728", "67108864");
3656                 if ($vga->{memory}) {
3657                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3658                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3659                 }
3660                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3661                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3662             }
3663         }
3664
3665         my $pciaddr = print_pci_addr("spice", $bridges);
3666
3667         my $nodename = PVE::INotify::nodename();
3668         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3669         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3670         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3671         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3672         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3673
3674         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3675
3676         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3677         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3678         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3679     }
3680
3681     # enable balloon by default, unless explicitly disabled
3682     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3683         $pciaddr = print_pci_addr("balloon0", $bridges);
3684         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3685     }
3686
3687     if ($conf->{watchdog}) {
3688         my $wdopts = parse_watchdog($conf->{watchdog});
3689         $pciaddr = print_pci_addr("watchdog", $bridges);
3690         my $watchdog = $wdopts->{model} || 'i6300esb';
3691         push @$devices, '-device', "$watchdog$pciaddr";
3692         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3693     }
3694
3695     my $vollist = [];
3696     my $scsicontroller = {};
3697     my $ahcicontroller = {};
3698     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3699
3700     # Add iscsi initiator name if available
3701     if (my $initiator = get_initiator_name()) {
3702         push @$devices, '-iscsi', "initiator-name=$initiator";
3703     }
3704
3705     foreach_drive($conf, sub {
3706         my ($ds, $drive) = @_;
3707
3708         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3709             push @$vollist, $drive->{file};
3710         }
3711
3712         # ignore efidisk here, already added in bios/fw handling code above
3713         return if $drive->{interface} eq 'efidisk';
3714
3715         $use_virtio = 1 if $ds =~ m/^virtio/;
3716
3717         if (drive_is_cdrom ($drive)) {
3718             if ($bootindex_hash->{d}) {
3719                 $drive->{bootindex} = $bootindex_hash->{d};
3720                 $bootindex_hash->{d} += 1;
3721             }
3722         } else {
3723             if ($bootindex_hash->{c}) {
3724                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3725                 $bootindex_hash->{c} += 1;
3726             }
3727         }
3728
3729         if($drive->{interface} eq 'virtio'){
3730            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3731         }
3732
3733         if ($drive->{interface} eq 'scsi') {
3734
3735             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3736
3737             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3738             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3739
3740             my $iothread = '';
3741             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3742                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3743                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3744             } elsif ($drive->{iothread}) {
3745                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3746             }
3747
3748             my $queues = '';
3749             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3750                 $queues = ",num_queues=$drive->{queues}";
3751             }
3752
3753             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3754             $scsicontroller->{$controller}=1;
3755         }
3756
3757         if ($drive->{interface} eq 'sata') {
3758            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3759            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3760            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3761            $ahcicontroller->{$controller}=1;
3762         }
3763
3764         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3765         push @$devices, '-drive',$drive_cmd;
3766         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3767     });
3768
3769     for (my $i = 0; $i < $MAX_NETS; $i++) {
3770          next if !$conf->{"net$i"};
3771          my $d = parse_net($conf->{"net$i"});
3772          next if !$d;
3773
3774          $use_virtio = 1 if $d->{model} eq 'virtio';
3775
3776          if ($bootindex_hash->{n}) {
3777             $d->{bootindex} = $bootindex_hash->{n};
3778             $bootindex_hash->{n} += 1;
3779          }
3780
3781          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3782          push @$devices, '-netdev', $netdevfull;
3783
3784          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3785          push @$devices, '-device', $netdevicefull;
3786     }
3787
3788     if (!$q35) {
3789         # add pci bridges
3790         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3791            $bridges->{1} = 1;
3792            $bridges->{2} = 1;
3793         }
3794
3795         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3796
3797         while (my ($k, $v) = each %$bridges) {
3798             $pciaddr = print_pci_addr("pci.$k");
3799             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3800         }
3801     }
3802
3803     # add custom args
3804     if ($conf->{args}) {
3805         my $aa = PVE::Tools::split_args($conf->{args});
3806         push @$cmd, @$aa;
3807     }
3808
3809     push @$cmd, @$devices;
3810     push @$cmd, '-rtc', join(',', @$rtcFlags)
3811         if scalar(@$rtcFlags);
3812     push @$cmd, '-machine', join(',', @$machineFlags)
3813         if scalar(@$machineFlags);
3814     push @$cmd, '-global', join(',', @$globalFlags)
3815         if scalar(@$globalFlags);
3816
3817     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3818 }
3819
3820 sub vnc_socket {
3821     my ($vmid) = @_;
3822     return "${var_run_tmpdir}/$vmid.vnc";
3823 }
3824
3825 sub spice_port {
3826     my ($vmid) = @_;
3827
3828     my $res = vm_mon_cmd($vmid, 'query-spice');
3829
3830     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3831 }
3832
3833 sub qmp_socket {
3834     my ($vmid, $qga, $name) = @_;
3835     my $sockettype = $qga ? 'qga' : 'qmp';
3836     my $ext = $name ? '-'.$name : '';
3837     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3838 }
3839
3840 sub pidfile_name {
3841     my ($vmid) = @_;
3842     return "${var_run_tmpdir}/$vmid.pid";
3843 }
3844
3845 sub vm_devices_list {
3846     my ($vmid) = @_;
3847
3848     my $res = vm_mon_cmd($vmid, 'query-pci');
3849     my $devices_to_check = [];
3850     my $devices = {};
3851     foreach my $pcibus (@$res) {
3852         push @$devices_to_check, @{$pcibus->{devices}},
3853     }
3854
3855     while (@$devices_to_check) {
3856         my $to_check = [];
3857         for my $d (@$devices_to_check) {
3858             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3859             next if !$d->{'pci_bridge'};
3860
3861             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3862             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3863         }
3864         $devices_to_check = $to_check;
3865     }
3866
3867     my $resblock = vm_mon_cmd($vmid, 'query-block');
3868     foreach my $block (@$resblock) {
3869         if($block->{device} =~ m/^drive-(\S+)/){
3870                 $devices->{$1} = 1;
3871         }
3872     }
3873
3874     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3875     foreach my $mice (@$resmice) {
3876         if ($mice->{name} eq 'QEMU HID Tablet') {
3877             $devices->{tablet} = 1;
3878             last;
3879         }
3880     }
3881
3882     # for usb devices there is no query-usb
3883     # but we can iterate over the entries in
3884     # qom-list path=/machine/peripheral
3885     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3886     foreach my $per (@$resperipheral) {
3887         if ($per->{name} =~ m/^usb\d+$/) {
3888             $devices->{$per->{name}} = 1;
3889         }
3890     }
3891
3892     return $devices;
3893 }
3894
3895 sub vm_deviceplug {
3896     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3897
3898     my $q35 = machine_type_is_q35($conf);
3899
3900     my $devices_list = vm_devices_list($vmid);
3901     return 1 if defined($devices_list->{$deviceid});
3902
3903     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3904
3905     if ($deviceid eq 'tablet') {
3906
3907         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3908
3909     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3910
3911         die "usb hotplug currently not reliable\n";
3912         # since we can't reliably hot unplug all added usb devices
3913         # and usb passthrough disables live migration
3914         # we disable usb hotplugging for now
3915         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3916
3917     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3918
3919         qemu_iothread_add($vmid, $deviceid, $device);
3920
3921         qemu_driveadd($storecfg, $vmid, $device);
3922         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3923
3924         qemu_deviceadd($vmid, $devicefull);
3925         eval { qemu_deviceaddverify($vmid, $deviceid); };
3926         if (my $err = $@) {
3927             eval { qemu_drivedel($vmid, $deviceid); };
3928             warn $@ if $@;
3929             die $err;
3930         }
3931
3932     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3933
3934
3935         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3936         my $pciaddr = print_pci_addr($deviceid);
3937         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3938
3939         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3940
3941         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3942             qemu_iothread_add($vmid, $deviceid, $device);
3943             $devicefull .= ",iothread=iothread-$deviceid";
3944         }
3945
3946         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3947             $devicefull .= ",num_queues=$device->{queues}";
3948         }
3949
3950         qemu_deviceadd($vmid, $devicefull);
3951         qemu_deviceaddverify($vmid, $deviceid);
3952
3953     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3954
3955         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3956         qemu_driveadd($storecfg, $vmid, $device);
3957
3958         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3959         eval { qemu_deviceadd($vmid, $devicefull); };
3960         if (my $err = $@) {
3961             eval { qemu_drivedel($vmid, $deviceid); };
3962             warn $@ if $@;
3963             die $err;
3964         }
3965
3966     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3967
3968         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3969
3970         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3971         my $use_old_bios_files = undef;
3972         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3973
3974         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3975         qemu_deviceadd($vmid, $netdevicefull);
3976         eval { qemu_deviceaddverify($vmid, $deviceid); };
3977         if (my $err = $@) {
3978             eval { qemu_netdevdel($vmid, $deviceid); };
3979             warn $@ if $@;
3980             die $err;
3981         }
3982
3983     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3984
3985         my $bridgeid = $2;
3986         my $pciaddr = print_pci_addr($deviceid);
3987         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3988
3989         qemu_deviceadd($vmid, $devicefull);
3990         qemu_deviceaddverify($vmid, $deviceid);
3991
3992     } else {
3993         die "can't hotplug device '$deviceid'\n";
3994     }
3995
3996     return 1;
3997 }
3998
3999 # fixme: this should raise exceptions on error!
4000 sub vm_deviceunplug {
4001     my ($vmid, $conf, $deviceid) = @_;
4002
4003     my $devices_list = vm_devices_list($vmid);
4004     return 1 if !defined($devices_list->{$deviceid});
4005
4006     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4007
4008     if ($deviceid eq 'tablet') {
4009
4010         qemu_devicedel($vmid, $deviceid);
4011
4012     } elsif ($deviceid =~ m/^usb\d+$/) {
4013
4014         die "usb hotplug currently not reliable\n";
4015         # when unplugging usb devices this way,
4016         # there may be remaining usb controllers/hubs
4017         # so we disable it for now
4018         qemu_devicedel($vmid, $deviceid);
4019         qemu_devicedelverify($vmid, $deviceid);
4020
4021     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4022
4023         qemu_devicedel($vmid, $deviceid);
4024         qemu_devicedelverify($vmid, $deviceid);
4025         qemu_drivedel($vmid, $deviceid);
4026         qemu_iothread_del($conf, $vmid, $deviceid);
4027
4028     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4029
4030         qemu_devicedel($vmid, $deviceid);
4031         qemu_devicedelverify($vmid, $deviceid);
4032         qemu_iothread_del($conf, $vmid, $deviceid);
4033
4034     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4035
4036         qemu_devicedel($vmid, $deviceid);
4037         qemu_drivedel($vmid, $deviceid);
4038         qemu_deletescsihw($conf, $vmid, $deviceid);
4039
4040     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4041
4042         qemu_devicedel($vmid, $deviceid);
4043         qemu_devicedelverify($vmid, $deviceid);
4044         qemu_netdevdel($vmid, $deviceid);
4045
4046     } else {
4047         die "can't unplug device '$deviceid'\n";
4048     }
4049
4050     return 1;
4051 }
4052
4053 sub qemu_deviceadd {
4054     my ($vmid, $devicefull) = @_;
4055
4056     $devicefull = "driver=".$devicefull;
4057     my %options =  split(/[=,]/, $devicefull);
4058
4059     vm_mon_cmd($vmid, "device_add" , %options);
4060 }
4061
4062 sub qemu_devicedel {
4063     my ($vmid, $deviceid) = @_;
4064
4065     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4066 }
4067
4068 sub qemu_iothread_add {
4069     my($vmid, $deviceid, $device) = @_;
4070
4071     if ($device->{iothread}) {
4072         my $iothreads = vm_iothreads_list($vmid);
4073         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4074     }
4075 }
4076
4077 sub qemu_iothread_del {
4078     my($conf, $vmid, $deviceid) = @_;
4079
4080     my $device = parse_drive($deviceid, $conf->{$deviceid});
4081     if ($device->{iothread}) {
4082         my $iothreads = vm_iothreads_list($vmid);
4083         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4084     }
4085 }
4086
4087 sub qemu_objectadd {
4088     my($vmid, $objectid, $qomtype) = @_;
4089
4090     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4091
4092     return 1;
4093 }
4094
4095 sub qemu_objectdel {
4096     my($vmid, $objectid) = @_;
4097
4098     vm_mon_cmd($vmid, "object-del", id => $objectid);
4099
4100     return 1;
4101 }
4102
4103 sub qemu_driveadd {
4104     my ($storecfg, $vmid, $device) = @_;
4105
4106     my $drive = print_drive_full($storecfg, $vmid, $device);
4107     $drive =~ s/\\/\\\\/g;
4108     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4109
4110     # If the command succeeds qemu prints: "OK"
4111     return 1 if $ret =~ m/OK/s;
4112
4113     die "adding drive failed: $ret\n";
4114 }
4115
4116 sub qemu_drivedel {
4117     my($vmid, $deviceid) = @_;
4118
4119     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4120     $ret =~ s/^\s+//;
4121
4122     return 1 if $ret eq "";
4123
4124     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4125     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4126
4127     die "deleting drive $deviceid failed : $ret\n";
4128 }
4129
4130 sub qemu_deviceaddverify {
4131     my ($vmid, $deviceid) = @_;
4132
4133     for (my $i = 0; $i <= 5; $i++) {
4134          my $devices_list = vm_devices_list($vmid);
4135          return 1 if defined($devices_list->{$deviceid});
4136          sleep 1;
4137     }
4138
4139     die "error on hotplug device '$deviceid'\n";
4140 }
4141
4142
4143 sub qemu_devicedelverify {
4144     my ($vmid, $deviceid) = @_;
4145
4146     # need to verify that the device is correctly removed as device_del
4147     # is async and empty return is not reliable
4148
4149     for (my $i = 0; $i <= 5; $i++) {
4150          my $devices_list = vm_devices_list($vmid);
4151          return 1 if !defined($devices_list->{$deviceid});
4152          sleep 1;
4153     }
4154
4155     die "error on hot-unplugging device '$deviceid'\n";
4156 }
4157
4158 sub qemu_findorcreatescsihw {
4159     my ($storecfg, $conf, $vmid, $device) = @_;
4160
4161     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4162
4163     my $scsihwid="$controller_prefix$controller";
4164     my $devices_list = vm_devices_list($vmid);
4165
4166     if(!defined($devices_list->{$scsihwid})) {
4167         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4168     }
4169
4170     return 1;
4171 }
4172
4173 sub qemu_deletescsihw {
4174     my ($conf, $vmid, $opt) = @_;
4175
4176     my $device = parse_drive($opt, $conf->{$opt});
4177
4178     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4179         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4180         return 1;
4181     }
4182
4183     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4184
4185     my $devices_list = vm_devices_list($vmid);
4186     foreach my $opt (keys %{$devices_list}) {
4187         if (PVE::QemuServer::is_valid_drivename($opt)) {
4188             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4189             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4190                 return 1;
4191             }
4192         }
4193     }
4194
4195     my $scsihwid="scsihw$controller";
4196
4197     vm_deviceunplug($vmid, $conf, $scsihwid);
4198
4199     return 1;
4200 }
4201
4202 sub qemu_add_pci_bridge {
4203     my ($storecfg, $conf, $vmid, $device) = @_;
4204
4205     my $bridges = {};
4206
4207     my $bridgeid;
4208
4209     print_pci_addr($device, $bridges);
4210
4211     while (my ($k, $v) = each %$bridges) {
4212         $bridgeid = $k;
4213     }
4214     return 1 if !defined($bridgeid) || $bridgeid < 1;
4215
4216     my $bridge = "pci.$bridgeid";
4217     my $devices_list = vm_devices_list($vmid);
4218
4219     if (!defined($devices_list->{$bridge})) {
4220         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4221     }
4222
4223     return 1;
4224 }
4225
4226 sub qemu_set_link_status {
4227     my ($vmid, $device, $up) = @_;
4228
4229     vm_mon_cmd($vmid, "set_link", name => $device,
4230                up => $up ? JSON::true : JSON::false);
4231 }
4232
4233 sub qemu_netdevadd {
4234     my ($vmid, $conf, $device, $deviceid) = @_;
4235
4236     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4237     my %options =  split(/[=,]/, $netdev);
4238
4239     vm_mon_cmd($vmid, "netdev_add",  %options);
4240     return 1;
4241 }
4242
4243 sub qemu_netdevdel {
4244     my ($vmid, $deviceid) = @_;
4245
4246     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4247 }
4248
4249 sub qemu_usb_hotplug {
4250     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4251
4252     return if !$device;
4253
4254     # remove the old one first
4255     vm_deviceunplug($vmid, $conf, $deviceid);
4256
4257     # check if xhci controller is necessary and available
4258     if ($device->{usb3}) {
4259
4260         my $devicelist = vm_devices_list($vmid);
4261
4262         if (!$devicelist->{xhci}) {
4263             my $pciaddr = print_pci_addr("xhci");
4264             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4265         }
4266     }
4267     my $d = parse_usb_device($device->{host});
4268     $d->{usb3} = $device->{usb3};
4269
4270     # add the new one
4271     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4272 }
4273
4274 sub qemu_cpu_hotplug {
4275     my ($vmid, $conf, $vcpus) = @_;
4276
4277     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4278
4279     my $sockets = 1;
4280     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4281     $sockets = $conf->{sockets} if  $conf->{sockets};
4282     my $cores = $conf->{cores} || 1;
4283     my $maxcpus = $sockets * $cores;
4284
4285     $vcpus = $maxcpus if !$vcpus;
4286
4287     die "you can't add more vcpus than maxcpus\n"
4288         if $vcpus > $maxcpus;
4289
4290     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4291
4292     if ($vcpus < $currentvcpus) {
4293
4294         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4295
4296             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4297                 qemu_devicedel($vmid, "cpu$i");
4298                 my $retry = 0;
4299                 my $currentrunningvcpus = undef;
4300                 while (1) {
4301                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4302                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4303                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4304                     $retry++;
4305                     sleep 1;
4306                 }
4307                 #update conf after each succesfull cpu unplug
4308                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4309                 PVE::QemuConfig->write_config($vmid, $conf);
4310             }
4311         } else {
4312             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4313         }
4314
4315         return;
4316     }
4317
4318     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4319     die "vcpus in running vm does not match its configuration\n"
4320         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4321
4322     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4323
4324         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4325             my $cpustr = print_cpu_device($conf, $i);
4326             qemu_deviceadd($vmid, $cpustr);
4327
4328             my $retry = 0;
4329             my $currentrunningvcpus = undef;
4330             while (1) {
4331                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4332                 last if scalar(@{$currentrunningvcpus}) == $i;
4333                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4334                 sleep 1;
4335                 $retry++;
4336             }
4337             #update conf after each succesfull cpu hotplug
4338             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4339             PVE::QemuConfig->write_config($vmid, $conf);
4340         }
4341     } else {
4342
4343         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4344             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4345         }
4346     }
4347 }
4348
4349 sub qemu_block_set_io_throttle {
4350     my ($vmid, $deviceid,
4351         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4352         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4353         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4354         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4355
4356     return if !check_running($vmid) ;
4357
4358     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4359         bps => int($bps),
4360         bps_rd => int($bps_rd),
4361         bps_wr => int($bps_wr),
4362         iops => int($iops),
4363         iops_rd => int($iops_rd),
4364         iops_wr => int($iops_wr),
4365         bps_max => int($bps_max),
4366         bps_rd_max => int($bps_rd_max),
4367         bps_wr_max => int($bps_wr_max),
4368         iops_max => int($iops_max),
4369         iops_rd_max => int($iops_rd_max),
4370         iops_wr_max => int($iops_wr_max),
4371         bps_max_length => int($bps_max_length),
4372         bps_rd_max_length => int($bps_rd_max_length),
4373         bps_wr_max_length => int($bps_wr_max_length),
4374         iops_max_length => int($iops_max_length),
4375         iops_rd_max_length => int($iops_rd_max_length),
4376         iops_wr_max_length => int($iops_wr_max_length),
4377     );
4378
4379 }
4380
4381 # old code, only used to shutdown old VM after update
4382 sub __read_avail {
4383     my ($fh, $timeout) = @_;
4384
4385     my $sel = new IO::Select;
4386     $sel->add($fh);
4387
4388     my $res = '';
4389     my $buf;
4390
4391     my @ready;
4392     while (scalar (@ready = $sel->can_read($timeout))) {
4393         my $count;
4394         if ($count = $fh->sysread($buf, 8192)) {
4395             if ($buf =~ /^(.*)\(qemu\) $/s) {
4396                 $res .= $1;
4397                 last;
4398             } else {
4399                 $res .= $buf;
4400             }
4401         } else {
4402             if (!defined($count)) {
4403                 die "$!\n";
4404             }
4405             last;
4406         }
4407     }
4408
4409     die "monitor read timeout\n" if !scalar(@ready);
4410
4411     return $res;
4412 }
4413
4414 sub qemu_block_resize {
4415     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4416
4417     my $running = check_running($vmid);
4418
4419     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4420
4421     return if !$running;
4422
4423     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4424
4425 }
4426
4427 sub qemu_volume_snapshot {
4428     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4429
4430     my $running = check_running($vmid);
4431
4432     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4433         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4434     } else {
4435         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4436     }
4437 }
4438
4439 sub qemu_volume_snapshot_delete {
4440     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4441
4442     my $running = check_running($vmid);
4443
4444     if($running) {
4445
4446         $running = undef;
4447         my $conf = PVE::QemuConfig->load_config($vmid);
4448         foreach_drive($conf, sub {
4449             my ($ds, $drive) = @_;
4450             $running = 1 if $drive->{file} eq $volid;
4451         });
4452     }
4453
4454     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4455         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4456     } else {
4457         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4458     }
4459 }
4460
4461 sub set_migration_caps {
4462     my ($vmid) = @_;
4463
4464     my $cap_ref = [];
4465
4466     my $enabled_cap = {
4467         "auto-converge" => 1,
4468         "xbzrle" => 1,
4469         "x-rdma-pin-all" => 0,
4470         "zero-blocks" => 0,
4471         "compress" => 0
4472     };
4473
4474     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4475
4476     for my $supported_capability (@$supported_capabilities) {
4477         push @$cap_ref, {
4478             capability => $supported_capability->{capability},
4479             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4480         };
4481     }
4482
4483     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4484 }
4485
4486 my $fast_plug_option = {
4487     'lock' => 1,
4488     'name' => 1,
4489     'onboot' => 1,
4490     'shares' => 1,
4491     'startup' => 1,
4492     'description' => 1,
4493     'protection' => 1,
4494     'vmstatestorage' => 1,
4495 };
4496
4497 # hotplug changes in [PENDING]
4498 # $selection hash can be used to only apply specified options, for
4499 # example: { cores => 1 } (only apply changed 'cores')
4500 # $errors ref is used to return error messages
4501 sub vmconfig_hotplug_pending {
4502     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4503
4504     my $defaults = load_defaults();
4505
4506     # commit values which do not have any impact on running VM first
4507     # Note: those option cannot raise errors, we we do not care about
4508     # $selection and always apply them.
4509
4510     my $add_error = sub {
4511         my ($opt, $msg) = @_;
4512         $errors->{$opt} = "hotplug problem - $msg";
4513     };
4514
4515     my $changes = 0;
4516     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4517         if ($fast_plug_option->{$opt}) {
4518             $conf->{$opt} = $conf->{pending}->{$opt};
4519             delete $conf->{pending}->{$opt};
4520             $changes = 1;
4521         }
4522     }
4523
4524     if ($changes) {
4525         PVE::QemuConfig->write_config($vmid, $conf);
4526         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4527     }
4528
4529     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4530
4531     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4532     while (my ($opt, $force) = each %$pending_delete_hash) {
4533         next if $selection && !$selection->{$opt};
4534         eval {
4535             if ($opt eq 'hotplug') {
4536                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4537             } elsif ($opt eq 'tablet') {
4538                 die "skip\n" if !$hotplug_features->{usb};
4539                 if ($defaults->{tablet}) {
4540                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4541                 } else {
4542                     vm_deviceunplug($vmid, $conf, $opt);
4543                 }
4544             } elsif ($opt =~ m/^usb\d+/) {
4545                 die "skip\n";
4546                 # since we cannot reliably hot unplug usb devices
4547                 # we are disabling it
4548                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4549                 vm_deviceunplug($vmid, $conf, $opt);
4550             } elsif ($opt eq 'vcpus') {
4551                 die "skip\n" if !$hotplug_features->{cpu};
4552                 qemu_cpu_hotplug($vmid, $conf, undef);
4553             } elsif ($opt eq 'balloon') {
4554                 # enable balloon device is not hotpluggable
4555                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4556                 # here we reset the ballooning value to memory
4557                 my $balloon = $conf->{memory} || $defaults->{memory};
4558                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4559             } elsif ($fast_plug_option->{$opt}) {
4560                 # do nothing
4561             } elsif ($opt =~ m/^net(\d+)$/) {
4562                 die "skip\n" if !$hotplug_features->{network};
4563                 vm_deviceunplug($vmid, $conf, $opt);
4564             } elsif (is_valid_drivename($opt)) {
4565                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4566                 vm_deviceunplug($vmid, $conf, $opt);
4567                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4568             } elsif ($opt =~ m/^memory$/) {
4569                 die "skip\n" if !$hotplug_features->{memory};
4570                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4571             } elsif ($opt eq 'cpuunits') {
4572                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4573             } elsif ($opt eq 'cpulimit') {
4574                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4575             } else {
4576                 die "skip\n";
4577             }
4578         };
4579         if (my $err = $@) {
4580             &$add_error($opt, $err) if $err ne "skip\n";
4581         } else {
4582             # save new config if hotplug was successful
4583             delete $conf->{$opt};
4584             vmconfig_undelete_pending_option($conf, $opt);
4585             PVE::QemuConfig->write_config($vmid, $conf);
4586             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4587         }
4588     }
4589
4590     my $apply_pending_cloudinit;
4591     $apply_pending_cloudinit = sub {
4592         my ($key, $value) = @_;
4593         $apply_pending_cloudinit = sub {}; # once is enough
4594
4595         my @cloudinit_opts = keys %$confdesc_cloudinit;
4596         foreach my $opt (keys %{$conf->{pending}}) {
4597             next if !grep { $_ eq $opt } @cloudinit_opts;
4598             $conf->{$opt} = delete $conf->{pending}->{$opt};
4599         }
4600
4601         my $new_conf = { %$conf };
4602         $new_conf->{$key} = $value;
4603         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4604     };
4605
4606     foreach my $opt (keys %{$conf->{pending}}) {
4607         next if $selection && !$selection->{$opt};
4608         my $value = $conf->{pending}->{$opt};
4609         eval {
4610             if ($opt eq 'hotplug') {
4611                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4612             } elsif ($opt eq 'tablet') {
4613                 die "skip\n" if !$hotplug_features->{usb};
4614                 if ($value == 1) {
4615                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4616                 } elsif ($value == 0) {
4617                     vm_deviceunplug($vmid, $conf, $opt);
4618                 }
4619             } elsif ($opt =~ m/^usb\d+$/) {
4620                 die "skip\n";
4621                 # since we cannot reliably hot unplug usb devices
4622                 # we are disabling it
4623                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4624                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4625                 die "skip\n" if !$d;
4626                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4627             } elsif ($opt eq 'vcpus') {
4628                 die "skip\n" if !$hotplug_features->{cpu};
4629                 qemu_cpu_hotplug($vmid, $conf, $value);
4630             } elsif ($opt eq 'balloon') {
4631                 # enable/disable balloning device is not hotpluggable
4632                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4633                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4634                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4635
4636                 # allow manual ballooning if shares is set to zero
4637                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4638                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4639                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4640                 }
4641             } elsif ($opt =~ m/^net(\d+)$/) {
4642                 # some changes can be done without hotplug
4643                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4644                                     $vmid, $opt, $value);
4645             } elsif (is_valid_drivename($opt)) {
4646                 # some changes can be done without hotplug
4647                 my $drive = parse_drive($opt, $value);
4648                 if (drive_is_cloudinit($drive)) {
4649                     &$apply_pending_cloudinit($opt, $value);
4650                 }
4651                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4652                                      $vmid, $opt, $value, 1);
4653             } elsif ($opt =~ m/^memory$/) { #dimms
4654                 die "skip\n" if !$hotplug_features->{memory};
4655                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4656             } elsif ($opt eq 'cpuunits') {
4657                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4658             } elsif ($opt eq 'cpulimit') {
4659                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4660                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4661             } else {
4662                 die "skip\n";  # skip non-hot-pluggable options
4663             }
4664         };
4665         if (my $err = $@) {
4666             &$add_error($opt, $err) if $err ne "skip\n";
4667         } else {
4668             # save new config if hotplug was successful
4669             $conf->{$opt} = $value;
4670             delete $conf->{pending}->{$opt};
4671             PVE::QemuConfig->write_config($vmid, $conf);
4672             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4673         }
4674     }
4675 }
4676
4677 sub try_deallocate_drive {
4678     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4679
4680     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4681         my $volid = $drive->{file};
4682         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4683             my $sid = PVE::Storage::parse_volume_id($volid);
4684             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4685
4686             # check if the disk is really unused
4687             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4688                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4689             PVE::Storage::vdisk_free($storecfg, $volid);
4690             return 1;
4691         } else {
4692             # If vm is not owner of this disk remove from config
4693             return 1;
4694         }
4695     }
4696
4697     return undef;
4698 }
4699
4700 sub vmconfig_delete_or_detach_drive {
4701     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4702
4703     my $drive = parse_drive($opt, $conf->{$opt});
4704
4705     my $rpcenv = PVE::RPCEnvironment::get();
4706     my $authuser = $rpcenv->get_user();
4707
4708     if ($force) {
4709         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4710         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4711     } else {
4712         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4713     }
4714 }
4715
4716 sub vmconfig_apply_pending {
4717     my ($vmid, $conf, $storecfg) = @_;
4718
4719     # cold plug
4720
4721     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4722     while (my ($opt, $force) = each %$pending_delete_hash) {
4723         die "internal error" if $opt =~ m/^unused/;
4724         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4725         if (!defined($conf->{$opt})) {
4726             vmconfig_undelete_pending_option($conf, $opt);
4727             PVE::QemuConfig->write_config($vmid, $conf);
4728         } elsif (is_valid_drivename($opt)) {
4729             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4730             vmconfig_undelete_pending_option($conf, $opt);
4731             delete $conf->{$opt};
4732             PVE::QemuConfig->write_config($vmid, $conf);
4733         } else {
4734             vmconfig_undelete_pending_option($conf, $opt);
4735             delete $conf->{$opt};
4736             PVE::QemuConfig->write_config($vmid, $conf);
4737         }
4738     }
4739
4740     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4741
4742     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4743         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4744
4745         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4746             # skip if nothing changed
4747         } elsif (is_valid_drivename($opt)) {
4748             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4749                 if defined($conf->{$opt});
4750             $conf->{$opt} = $conf->{pending}->{$opt};
4751         } else {
4752             $conf->{$opt} = $conf->{pending}->{$opt};
4753         }
4754
4755         delete $conf->{pending}->{$opt};
4756         PVE::QemuConfig->write_config($vmid, $conf);
4757     }
4758 }
4759
4760 my $safe_num_ne = sub {
4761     my ($a, $b) = @_;
4762
4763     return 0 if !defined($a) && !defined($b);
4764     return 1 if !defined($a);
4765     return 1 if !defined($b);
4766
4767     return $a != $b;
4768 };
4769
4770 my $safe_string_ne = sub {
4771     my ($a, $b) = @_;
4772
4773     return 0 if !defined($a) && !defined($b);
4774     return 1 if !defined($a);
4775     return 1 if !defined($b);
4776
4777     return $a ne $b;
4778 };
4779
4780 sub vmconfig_update_net {
4781     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4782
4783     my $newnet = parse_net($value);
4784
4785     if ($conf->{$opt}) {
4786         my $oldnet = parse_net($conf->{$opt});
4787
4788         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4789             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4790             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4791             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4792
4793             # for non online change, we try to hot-unplug
4794             die "skip\n" if !$hotplug;
4795             vm_deviceunplug($vmid, $conf, $opt);
4796         } else {
4797
4798             die "internal error" if $opt !~ m/net(\d+)/;
4799             my $iface = "tap${vmid}i$1";
4800
4801             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4802                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4803                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4804                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4805                 PVE::Network::tap_unplug($iface);
4806                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4807             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4808                 # Rate can be applied on its own but any change above needs to
4809                 # include the rate in tap_plug since OVS resets everything.
4810                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4811             }
4812
4813             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4814                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4815             }
4816
4817             return 1;
4818         }
4819     }
4820
4821     if ($hotplug) {
4822         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4823     } else {
4824         die "skip\n";
4825     }
4826 }
4827
4828 sub vmconfig_update_disk {
4829     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4830
4831     # fixme: do we need force?
4832
4833     my $drive = parse_drive($opt, $value);
4834
4835     if ($conf->{$opt}) {
4836
4837         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4838
4839             my $media = $drive->{media} || 'disk';
4840             my $oldmedia = $old_drive->{media} || 'disk';
4841             die "unable to change media type\n" if $media ne $oldmedia;
4842
4843             if (!drive_is_cdrom($old_drive)) {
4844
4845                 if ($drive->{file} ne $old_drive->{file}) {
4846
4847                     die "skip\n" if !$hotplug;
4848
4849                     # unplug and register as unused
4850                     vm_deviceunplug($vmid, $conf, $opt);
4851                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4852
4853                 } else {
4854                     # update existing disk
4855
4856                     # skip non hotpluggable value
4857                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4858                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4859                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4860                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4861                         die "skip\n";
4862                     }
4863
4864                     # apply throttle
4865                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4866                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4867                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4868                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4869                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4870                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4871                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4872                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4873                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4874                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4875                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4876                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4877                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4878                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4879                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4880                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4881                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4882                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4883
4884                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4885                                                    ($drive->{mbps} || 0)*1024*1024,
4886                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4887                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4888                                                    $drive->{iops} || 0,
4889                                                    $drive->{iops_rd} || 0,
4890                                                    $drive->{iops_wr} || 0,
4891                                                    ($drive->{mbps_max} || 0)*1024*1024,
4892                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4893                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4894                                                    $drive->{iops_max} || 0,
4895                                                    $drive->{iops_rd_max} || 0,
4896                                                    $drive->{iops_wr_max} || 0,
4897                                                    $drive->{bps_max_length} || 1,
4898                                                    $drive->{bps_rd_max_length} || 1,
4899                                                    $drive->{bps_wr_max_length} || 1,
4900                                                    $drive->{iops_max_length} || 1,
4901                                                    $drive->{iops_rd_max_length} || 1,
4902                                                    $drive->{iops_wr_max_length} || 1);
4903
4904                     }
4905
4906                     return 1;
4907                 }
4908
4909             } else { # cdrom
4910
4911                 if ($drive->{file} eq 'none') {
4912                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4913                     if (drive_is_cloudinit($old_drive)) {
4914                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4915                     }
4916                 } else {
4917                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4918                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4919                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4920                 }
4921
4922                 return 1;
4923             }
4924         }
4925     }
4926
4927     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4928     # hotplug new disks
4929     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4930     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4931 }
4932
4933 sub vm_start {
4934     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4935         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4936
4937     PVE::QemuConfig->lock_config($vmid, sub {
4938         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4939
4940         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4941
4942         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4943
4944         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4945
4946         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4947             vmconfig_apply_pending($vmid, $conf, $storecfg);
4948             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4949         }
4950
4951         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4952
4953         my $defaults = load_defaults();
4954
4955         # set environment variable useful inside network script
4956         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4957
4958         my $local_volumes = {};
4959
4960         if ($targetstorage) {
4961             foreach_drive($conf, sub {
4962                 my ($ds, $drive) = @_;
4963
4964                 return if drive_is_cdrom($drive);
4965
4966                 my $volid = $drive->{file};
4967
4968                 return if !$volid;
4969
4970                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4971
4972                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4973                 return if $scfg->{shared};
4974                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4975             });
4976
4977             my $format = undef;
4978
4979             foreach my $opt (sort keys %$local_volumes) {
4980
4981                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4982                 my $drive = parse_drive($opt, $conf->{$opt});
4983
4984                 #if remote storage is specified, use default format
4985                 if ($targetstorage && $targetstorage ne "1") {
4986                     $storeid = $targetstorage;
4987                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4988                     $format = $defFormat;
4989                 } else {
4990                     #else we use same format than original
4991                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4992                     $format = qemu_img_format($scfg, $volid);
4993                 }
4994
4995                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4996                 my $newdrive = $drive;
4997                 $newdrive->{format} = $format;
4998                 $newdrive->{file} = $newvolid;
4999                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5000                 $local_volumes->{$opt} = $drivestr;
5001                 #pass drive to conf for command line
5002                 $conf->{$opt} = $drivestr;
5003             }
5004         }
5005
5006         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5007
5008         my $migrate_port = 0;
5009         my $migrate_uri;
5010         if ($statefile) {
5011             if ($statefile eq 'tcp') {
5012                 my $localip = "localhost";
5013                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5014                 my $nodename = PVE::INotify::nodename();
5015
5016                 if (!defined($migration_type)) {
5017                     if (defined($datacenterconf->{migration}->{type})) {
5018                         $migration_type = $datacenterconf->{migration}->{type};
5019                     } else {
5020                         $migration_type = 'secure';
5021                     }
5022                 }
5023
5024                 if ($migration_type eq 'insecure') {
5025                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5026                     if ($migrate_network_addr) {
5027                         $localip = $migrate_network_addr;
5028                     } else {
5029                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5030                     }
5031
5032                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5033                 }
5034
5035                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5036                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5037                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5038                 push @$cmd, '-incoming', $migrate_uri;
5039                 push @$cmd, '-S';
5040
5041             } elsif ($statefile eq 'unix') {
5042                 # should be default for secure migrations as a ssh TCP forward
5043                 # tunnel is not deterministic reliable ready and fails regurarly
5044                 # to set up in time, so use UNIX socket forwards
5045                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5046                 unlink $socket_addr;
5047
5048                 $migrate_uri = "unix:$socket_addr";
5049
5050                 push @$cmd, '-incoming', $migrate_uri;
5051                 push @$cmd, '-S';
5052
5053             } else {
5054                 push @$cmd, '-loadstate', $statefile;
5055             }
5056         } elsif ($paused) {
5057             push @$cmd, '-S';
5058         }
5059
5060         # host pci devices
5061         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5062           my $d = parse_hostpci($conf->{"hostpci$i"});
5063           next if !$d;
5064           my $pcidevices = $d->{pciid};
5065           foreach my $pcidevice (@$pcidevices) {
5066                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5067
5068                 my $info = pci_device_info("0000:$pciid");
5069                 die "IOMMU not present\n" if !check_iommu_support();
5070                 die "no pci device info for device '$pciid'\n" if !$info;
5071                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
5072                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
5073           }
5074         }
5075
5076         PVE::Storage::activate_volumes($storecfg, $vollist);
5077
5078         if (!check_running($vmid, 1)) {
5079             eval {
5080                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5081                     outfunc => sub {}, errfunc => sub {});
5082             };
5083         }
5084
5085         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5086                                                   : $defaults->{cpuunits};
5087
5088         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5089         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5090
5091         my %properties = (
5092             Slice => 'qemu.slice',
5093             KillMode => 'none',
5094             CPUShares => $cpuunits
5095         );
5096
5097         if (my $cpulimit = $conf->{cpulimit}) {
5098             $properties{CPUQuota} = int($cpulimit * 100);
5099         }
5100         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5101
5102         my $run_qemu = sub {
5103             PVE::Tools::run_fork sub {
5104                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5105                 run_command($cmd, %run_params);
5106             };
5107         };
5108
5109         if ($conf->{hugepages}) {
5110
5111             my $code = sub {
5112                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5113                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5114
5115                 PVE::QemuServer::Memory::hugepages_mount();
5116                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5117
5118                 eval { $run_qemu->() };
5119                 if (my $err = $@) {
5120                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5121                     die $err;
5122                 }
5123
5124                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5125             };
5126             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5127
5128         } else {
5129             eval { $run_qemu->() };
5130         }
5131
5132         if (my $err = $@) {
5133             # deactivate volumes if start fails
5134             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5135             die "start failed: $err";
5136         }
5137
5138         print "migration listens on $migrate_uri\n" if $migrate_uri;
5139
5140         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5141             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5142             warn $@ if $@;
5143         }
5144
5145         #start nbd server for storage migration
5146         if ($targetstorage) {
5147             my $nodename = PVE::INotify::nodename();
5148             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5149             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5150             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5151             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5152
5153             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5154
5155             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5156
5157             foreach my $opt (sort keys %$local_volumes) {
5158                 my $volid = $local_volumes->{$opt};
5159                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5160                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5161                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5162             }
5163         }
5164
5165         if ($migratedfrom) {
5166             eval {
5167                 set_migration_caps($vmid);
5168             };
5169             warn $@ if $@;
5170
5171             if ($spice_port) {
5172                 print "spice listens on port $spice_port\n";
5173                 if ($spice_ticket) {
5174                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5175                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5176                 }
5177             }
5178
5179         } else {
5180             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5181                 if !$statefile && $conf->{balloon};
5182
5183             foreach my $opt (keys %$conf) {
5184                 next if $opt !~  m/^net\d+$/;
5185                 my $nicconf = parse_net($conf->{$opt});
5186                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5187             }
5188         }
5189
5190         vm_mon_cmd_nocheck($vmid, 'qom-set',
5191                     path => "machine/peripheral/balloon0",
5192                     property => "guest-stats-polling-interval",
5193                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5194
5195     });
5196 }
5197
5198 sub vm_mon_cmd {
5199     my ($vmid, $execute, %params) = @_;
5200
5201     my $cmd = { execute => $execute, arguments => \%params };
5202     vm_qmp_command($vmid, $cmd);
5203 }
5204
5205 sub vm_mon_cmd_nocheck {
5206     my ($vmid, $execute, %params) = @_;
5207
5208     my $cmd = { execute => $execute, arguments => \%params };
5209     vm_qmp_command($vmid, $cmd, 1);
5210 }
5211
5212 sub vm_qmp_command {
5213     my ($vmid, $cmd, $nocheck) = @_;
5214
5215     my $res;
5216
5217     my $timeout;
5218     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5219         $timeout = $cmd->{arguments}->{timeout};
5220         delete $cmd->{arguments}->{timeout};
5221     }
5222
5223     eval {
5224         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5225         my $sname = qmp_socket($vmid);
5226         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5227             my $qmpclient = PVE::QMPClient->new();
5228
5229             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5230         } else {
5231             die "unable to open monitor socket\n";
5232         }
5233     };
5234     if (my $err = $@) {
5235         syslog("err", "VM $vmid qmp command failed - $err");
5236         die $err;
5237     }
5238
5239     return $res;
5240 }
5241
5242 sub vm_human_monitor_command {
5243     my ($vmid, $cmdline) = @_;
5244
5245     my $res;
5246
5247     my $cmd = {
5248         execute => 'human-monitor-command',
5249         arguments => { 'command-line' => $cmdline},
5250     };
5251
5252     return vm_qmp_command($vmid, $cmd);
5253 }
5254
5255 sub vm_commandline {
5256     my ($storecfg, $vmid) = @_;
5257
5258     my $conf = PVE::QemuConfig->load_config($vmid);
5259
5260     my $defaults = load_defaults();
5261
5262     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5263
5264     return PVE::Tools::cmd2string($cmd);
5265 }
5266
5267 sub vm_reset {
5268     my ($vmid, $skiplock) = @_;
5269
5270     PVE::QemuConfig->lock_config($vmid, sub {
5271
5272         my $conf = PVE::QemuConfig->load_config($vmid);
5273
5274         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5275
5276         vm_mon_cmd($vmid, "system_reset");
5277     });
5278 }
5279
5280 sub get_vm_volumes {
5281     my ($conf) = @_;
5282
5283     my $vollist = [];
5284     foreach_volid($conf, sub {
5285         my ($volid, $attr) = @_;
5286
5287         return if $volid =~ m|^/|;
5288
5289         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5290         return if !$sid;
5291
5292         push @$vollist, $volid;
5293     });
5294
5295     return $vollist;
5296 }
5297
5298 sub vm_stop_cleanup {
5299     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5300
5301     eval {
5302
5303         if (!$keepActive) {
5304             my $vollist = get_vm_volumes($conf);
5305             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5306         }
5307
5308         foreach my $ext (qw(mon qmp pid vnc qga)) {
5309             unlink "/var/run/qemu-server/${vmid}.$ext";
5310         }
5311
5312         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5313     };
5314     warn $@ if $@; # avoid errors - just warn
5315 }
5316
5317 # Note: use $nockeck to skip tests if VM configuration file exists.
5318 # We need that when migration VMs to other nodes (files already moved)
5319 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5320 sub vm_stop {
5321     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5322
5323     $force = 1 if !defined($force) && !$shutdown;
5324
5325     if ($migratedfrom){
5326         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5327         kill 15, $pid if $pid;
5328         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5329         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5330         return;
5331     }
5332
5333     PVE::QemuConfig->lock_config($vmid, sub {
5334
5335         my $pid = check_running($vmid, $nocheck);
5336         return if !$pid;
5337
5338         my $conf;
5339         if (!$nocheck) {
5340             $conf = PVE::QemuConfig->load_config($vmid);
5341             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5342             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5343                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5344                 $timeout = $opts->{down} if $opts->{down};
5345             }
5346         }
5347
5348         $timeout = 60 if !defined($timeout);
5349
5350         eval {
5351             if ($shutdown) {
5352                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5353                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5354                 } else {
5355                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5356                 }
5357             } else {
5358                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5359             }
5360         };
5361         my $err = $@;
5362
5363         if (!$err) {
5364             my $count = 0;
5365             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5366                 $count++;
5367                 sleep 1;
5368             }
5369
5370             if ($count >= $timeout) {
5371                 if ($force) {
5372                     warn "VM still running - terminating now with SIGTERM\n";
5373                     kill 15, $pid;
5374                 } else {
5375                     die "VM quit/powerdown failed - got timeout\n";
5376                 }
5377             } else {
5378                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5379                 return;
5380             }
5381         } else {
5382             if ($force) {
5383                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5384                 kill 15, $pid;
5385             } else {
5386                 die "VM quit/powerdown failed\n";
5387             }
5388         }
5389
5390         # wait again
5391         $timeout = 10;
5392
5393         my $count = 0;
5394         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5395             $count++;
5396             sleep 1;
5397         }
5398
5399         if ($count >= $timeout) {
5400             warn "VM still running - terminating now with SIGKILL\n";
5401             kill 9, $pid;
5402             sleep 1;
5403         }
5404
5405         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5406    });
5407 }
5408
5409 sub vm_suspend {
5410     my ($vmid, $skiplock) = @_;
5411
5412     PVE::QemuConfig->lock_config($vmid, sub {
5413
5414         my $conf = PVE::QemuConfig->load_config($vmid);
5415
5416         PVE::QemuConfig->check_lock($conf)
5417             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5418
5419         vm_mon_cmd($vmid, "stop");
5420     });
5421 }
5422
5423 sub vm_resume {
5424     my ($vmid, $skiplock, $nocheck) = @_;
5425
5426     PVE::QemuConfig->lock_config($vmid, sub {
5427
5428         my $res = vm_mon_cmd($vmid, 'query-status');
5429         my $resume_cmd = 'cont';
5430
5431         if ($res->{status} && $res->{status} eq 'suspended') {
5432             $resume_cmd = 'system_wakeup';
5433         }
5434
5435         if (!$nocheck) {
5436
5437             my $conf = PVE::QemuConfig->load_config($vmid);
5438
5439             PVE::QemuConfig->check_lock($conf)
5440                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5441
5442             vm_mon_cmd($vmid, $resume_cmd);
5443
5444         } else {
5445             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5446         }
5447     });
5448 }
5449
5450 sub vm_sendkey {
5451     my ($vmid, $skiplock, $key) = @_;
5452
5453     PVE::QemuConfig->lock_config($vmid, sub {
5454
5455         my $conf = PVE::QemuConfig->load_config($vmid);
5456
5457         # there is no qmp command, so we use the human monitor command
5458         vm_human_monitor_command($vmid, "sendkey $key");
5459     });
5460 }
5461
5462 sub vm_destroy {
5463     my ($storecfg, $vmid, $skiplock) = @_;
5464
5465     PVE::QemuConfig->lock_config($vmid, sub {
5466
5467         my $conf = PVE::QemuConfig->load_config($vmid);
5468
5469         if (!check_running($vmid)) {
5470             destroy_vm($storecfg, $vmid, undef, $skiplock);
5471         } else {
5472             die "VM $vmid is running - destroy failed\n";
5473         }
5474     });
5475 }
5476
5477 # pci helpers
5478
5479 sub file_write {
5480     my ($filename, $buf) = @_;
5481
5482     my $fh = IO::File->new($filename, "w");
5483     return undef if !$fh;
5484
5485     my $res = print $fh $buf;
5486
5487     $fh->close();
5488
5489     return $res;
5490 }
5491
5492 sub pci_device_info {
5493     my ($name) = @_;
5494
5495     my $res;
5496
5497     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5498     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5499
5500     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5501     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5502
5503     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5504     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5505
5506     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5507     return undef if !defined($product) || $product !~ s/^0x//;
5508
5509     $res = {
5510         name => $name,
5511         vendor => $vendor,
5512         product => $product,
5513         domain => $domain,
5514         bus => $bus,
5515         slot => $slot,
5516         func => $func,
5517         irq => $irq,
5518         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5519     };
5520
5521     return $res;
5522 }
5523
5524 sub pci_dev_reset {
5525     my ($dev) = @_;
5526
5527     my $name = $dev->{name};
5528
5529     my $fn = "$pcisysfs/devices/$name/reset";
5530
5531     return file_write($fn, "1");
5532 }
5533
5534 sub pci_dev_bind_to_vfio {
5535     my ($dev) = @_;
5536
5537     my $name = $dev->{name};
5538
5539     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5540
5541     if (!-d $vfio_basedir) {
5542         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5543     }
5544     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5545
5546     my $testdir = "$vfio_basedir/$name";
5547     return 1 if -d $testdir;
5548
5549     my $data = "$dev->{vendor} $dev->{product}";
5550     return undef if !file_write("$vfio_basedir/new_id", $data);
5551
5552     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5553     if (!file_write($fn, $name)) {
5554         return undef if -f $fn;
5555     }
5556
5557     $fn = "$vfio_basedir/bind";
5558     if (! -d $testdir) {
5559         return undef if !file_write($fn, $name);
5560     }
5561
5562     return -d $testdir;
5563 }
5564
5565 sub pci_dev_group_bind_to_vfio {
5566     my ($pciid) = @_;
5567
5568     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5569
5570     if (!-d $vfio_basedir) {
5571         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5572     }
5573     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5574
5575     # get IOMMU group devices
5576     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5577       my @devs = grep /^0000:/, readdir($D);
5578     closedir($D);
5579
5580     foreach my $pciid (@devs) {
5581         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5582
5583         # pci bridges, switches or root ports are not supported
5584         # they have a pci_bus subdirectory so skip them
5585         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5586
5587         my $info = pci_device_info($1);
5588         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5589     }
5590
5591     return 1;
5592 }
5593
5594 # vzdump restore implementaion
5595
5596 sub tar_archive_read_firstfile {
5597     my $archive = shift;
5598
5599     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5600
5601     # try to detect archive type first
5602     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5603         die "unable to open file '$archive'\n";
5604     my $firstfile = <$fh>;
5605     kill 15, $pid;
5606     close $fh;
5607
5608     die "ERROR: archive contaions no data\n" if !$firstfile;
5609     chomp $firstfile;
5610
5611     return $firstfile;
5612 }
5613
5614 sub tar_restore_cleanup {
5615     my ($storecfg, $statfile) = @_;
5616
5617     print STDERR "starting cleanup\n";
5618
5619     if (my $fd = IO::File->new($statfile, "r")) {
5620         while (defined(my $line = <$fd>)) {
5621             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5622                 my $volid = $2;
5623                 eval {
5624                     if ($volid =~ m|^/|) {
5625                         unlink $volid || die 'unlink failed\n';
5626                     } else {
5627                         PVE::Storage::vdisk_free($storecfg, $volid);
5628                     }
5629                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5630                 };
5631                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5632             } else {
5633                 print STDERR "unable to parse line in statfile - $line";
5634             }
5635         }
5636         $fd->close();
5637     }
5638 }
5639
5640 sub restore_archive {
5641     my ($archive, $vmid, $user, $opts) = @_;
5642
5643     my $format = $opts->{format};
5644     my $comp;
5645
5646     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5647         $format = 'tar' if !$format;
5648         $comp = 'gzip';
5649     } elsif ($archive =~ m/\.tar$/) {
5650         $format = 'tar' if !$format;
5651     } elsif ($archive =~ m/.tar.lzo$/) {
5652         $format = 'tar' if !$format;
5653         $comp = 'lzop';
5654     } elsif ($archive =~ m/\.vma$/) {
5655         $format = 'vma' if !$format;
5656     } elsif ($archive =~ m/\.vma\.gz$/) {
5657         $format = 'vma' if !$format;
5658         $comp = 'gzip';
5659     } elsif ($archive =~ m/\.vma\.lzo$/) {
5660         $format = 'vma' if !$format;
5661         $comp = 'lzop';
5662     } else {
5663         $format = 'vma' if !$format; # default
5664     }
5665
5666     # try to detect archive format
5667     if ($format eq 'tar') {
5668         return restore_tar_archive($archive, $vmid, $user, $opts);
5669     } else {
5670         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5671     }
5672 }
5673
5674 sub restore_update_config_line {
5675     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5676
5677     return if $line =~ m/^\#qmdump\#/;
5678     return if $line =~ m/^\#vzdump\#/;
5679     return if $line =~ m/^lock:/;
5680     return if $line =~ m/^unused\d+:/;
5681     return if $line =~ m/^parent:/;
5682     return if $line =~ m/^template:/; # restored VM is never a template
5683
5684     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5685     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5686         # try to convert old 1.X settings
5687         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5688         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5689             my ($model, $macaddr) = split(/\=/, $devconfig);
5690             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5691             my $net = {
5692                 model => $model,
5693                 bridge => "vmbr$ind",
5694                 macaddr => $macaddr,
5695             };
5696             my $netstr = print_net($net);
5697
5698             print $outfd "net$cookie->{netcount}: $netstr\n";
5699             $cookie->{netcount}++;
5700         }
5701     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5702         my ($id, $netstr) = ($1, $2);
5703         my $net = parse_net($netstr);
5704         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5705         $netstr = print_net($net);
5706         print $outfd "$id: $netstr\n";
5707     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5708         my $virtdev = $1;
5709         my $value = $3;
5710         my $di = parse_drive($virtdev, $value);
5711         if (defined($di->{backup}) && !$di->{backup}) {
5712             print $outfd "#$line";
5713         } elsif ($map->{$virtdev}) {
5714             delete $di->{format}; # format can change on restore
5715             $di->{file} = $map->{$virtdev};
5716             $value = print_drive($vmid, $di);
5717             print $outfd "$virtdev: $value\n";
5718         } else {
5719             print $outfd $line;
5720         }
5721     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5722         my $vmgenid = $1;
5723         if ($vmgenid ne '0') {
5724             # always generate a new vmgenid if there was a valid one setup
5725             $vmgenid = generate_uuid();
5726         }
5727         print $outfd "vmgenid: $vmgenid\n";
5728     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5729         my ($uuid, $uuid_str);
5730         UUID::generate($uuid);
5731         UUID::unparse($uuid, $uuid_str);
5732         my $smbios1 = parse_smbios1($2);
5733         $smbios1->{uuid} = $uuid_str;
5734         print $outfd $1.print_smbios1($smbios1)."\n";
5735     } else {
5736         print $outfd $line;
5737     }
5738 }
5739
5740 sub scan_volids {
5741     my ($cfg, $vmid) = @_;
5742
5743     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5744
5745     my $volid_hash = {};
5746     foreach my $storeid (keys %$info) {
5747         foreach my $item (@{$info->{$storeid}}) {
5748             next if !($item->{volid} && $item->{size});
5749             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5750             $volid_hash->{$item->{volid}} = $item;
5751         }
5752     }
5753
5754     return $volid_hash;
5755 }
5756
5757 sub is_volume_in_use {
5758     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5759
5760     my $path = PVE::Storage::path($storecfg, $volid);
5761
5762     my $scan_config = sub {
5763         my ($cref, $snapname) = @_;
5764
5765         foreach my $key (keys %$cref) {
5766             my $value = $cref->{$key};
5767             if (is_valid_drivename($key)) {
5768                 next if $skip_drive && $key eq $skip_drive;
5769                 my $drive = parse_drive($key, $value);
5770                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5771                 return 1 if $volid eq $drive->{file};
5772                 if ($drive->{file} =~ m!^/!) {
5773                     return 1 if $drive->{file} eq $path;
5774                 } else {
5775                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5776                     next if !$storeid;
5777                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5778                     next if !$scfg;
5779                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5780                 }
5781             }
5782         }
5783
5784         return 0;
5785     };
5786
5787     return 1 if &$scan_config($conf);
5788
5789     undef $skip_drive;
5790
5791     foreach my $snapname (keys %{$conf->{snapshots}}) {
5792         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5793     }
5794
5795     return 0;
5796 }
5797
5798 sub update_disksize {
5799     my ($vmid, $conf, $volid_hash) = @_;
5800
5801     my $changes;
5802     my $prefix = "VM $vmid:";
5803
5804     # used and unused disks
5805     my $referenced = {};
5806
5807     # Note: it is allowed to define multiple storages with same path (alias), so
5808     # we need to check both 'volid' and real 'path' (two different volid can point
5809     # to the same path).
5810
5811     my $referencedpath = {};
5812
5813     # update size info
5814     foreach my $opt (keys %$conf) {
5815         if (is_valid_drivename($opt)) {
5816             my $drive = parse_drive($opt, $conf->{$opt});
5817             my $volid = $drive->{file};
5818             next if !$volid;
5819
5820             $referenced->{$volid} = 1;
5821             if ($volid_hash->{$volid} &&
5822                 (my $path = $volid_hash->{$volid}->{path})) {
5823                 $referencedpath->{$path} = 1;
5824             }
5825
5826             next if drive_is_cdrom($drive);
5827             next if !$volid_hash->{$volid};
5828
5829             $drive->{size} = $volid_hash->{$volid}->{size};
5830             my $new = print_drive($vmid, $drive);
5831             if ($new ne $conf->{$opt}) {
5832                 $changes = 1;
5833                 $conf->{$opt} = $new;
5834                 print "$prefix update disk '$opt' information.\n";
5835             }
5836         }
5837     }
5838
5839     # remove 'unusedX' entry if volume is used
5840     foreach my $opt (keys %$conf) {
5841         next if $opt !~ m/^unused\d+$/;
5842         my $volid = $conf->{$opt};
5843         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5844         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5845             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5846             $changes = 1;
5847             delete $conf->{$opt};
5848         }
5849
5850         $referenced->{$volid} = 1;
5851         $referencedpath->{$path} = 1 if $path;
5852     }
5853
5854     foreach my $volid (sort keys %$volid_hash) {
5855         next if $volid =~ m/vm-$vmid-state-/;
5856         next if $referenced->{$volid};
5857         my $path = $volid_hash->{$volid}->{path};
5858         next if !$path; # just to be sure
5859         next if $referencedpath->{$path};
5860         $changes = 1;
5861         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5862         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5863         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5864     }
5865
5866     return $changes;
5867 }
5868
5869 sub rescan {
5870     my ($vmid, $nolock, $dryrun) = @_;
5871
5872     my $cfg = PVE::Storage::config();
5873
5874     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5875     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5876     foreach my $stor (keys %{$cfg->{ids}}) {
5877         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5878     }
5879
5880     print "rescan volumes...\n";
5881     my $volid_hash = scan_volids($cfg, $vmid);
5882
5883     my $updatefn =  sub {
5884         my ($vmid) = @_;
5885
5886         my $conf = PVE::QemuConfig->load_config($vmid);
5887
5888         PVE::QemuConfig->check_lock($conf);
5889
5890         my $vm_volids = {};
5891         foreach my $volid (keys %$volid_hash) {
5892             my $info = $volid_hash->{$volid};
5893             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5894         }
5895
5896         my $changes = update_disksize($vmid, $conf, $vm_volids);
5897
5898         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5899     };
5900
5901     if (defined($vmid)) {
5902         if ($nolock) {
5903             &$updatefn($vmid);
5904         } else {
5905             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5906         }
5907     } else {
5908         my $vmlist = config_list();
5909         foreach my $vmid (keys %$vmlist) {
5910             if ($nolock) {
5911                 &$updatefn($vmid);
5912             } else {
5913                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5914             }
5915         }
5916     }
5917 }
5918
5919 sub restore_vma_archive {
5920     my ($archive, $vmid, $user, $opts, $comp) = @_;
5921
5922     my $readfrom = $archive;
5923
5924     my $cfg = PVE::Storage::config();
5925     my $commands = [];
5926     my $bwlimit = $opts->{bwlimit};
5927
5928     my $dbg_cmdstring = '';
5929     my $add_pipe = sub {
5930         my ($cmd) = @_;
5931         push @$commands, $cmd;
5932         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5933         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5934         $readfrom = '-';
5935     };
5936
5937     my $input = undef;
5938     if ($archive eq '-') {
5939         $input = '<&STDIN';
5940     } else {
5941         # If we use a backup from a PVE defined storage we also consider that
5942         # storage's rate limit:
5943         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5944         if (defined($volid)) {
5945             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5946             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5947             if ($readlimit) {
5948                 print STDERR "applying read rate limit: $readlimit\n";
5949                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5950                 $add_pipe->($cstream);
5951             }
5952         }
5953     }
5954
5955     if ($comp) {
5956         my $cmd;
5957         if ($comp eq 'gzip') {
5958             $cmd = ['zcat', $readfrom];
5959         } elsif ($comp eq 'lzop') {
5960             $cmd = ['lzop', '-d', '-c', $readfrom];
5961         } else {
5962             die "unknown compression method '$comp'\n";
5963         }
5964         $add_pipe->($cmd);
5965     }
5966
5967     my $tmpdir = "/var/tmp/vzdumptmp$$";
5968     rmtree $tmpdir;
5969
5970     # disable interrupts (always do cleanups)
5971     local $SIG{INT} =
5972         local $SIG{TERM} =
5973         local $SIG{QUIT} =
5974         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5975
5976     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5977     POSIX::mkfifo($mapfifo, 0600);
5978     my $fifofh;
5979
5980     my $openfifo = sub {
5981         open($fifofh, '>', $mapfifo) || die $!;
5982     };
5983
5984     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5985
5986     my $oldtimeout;
5987     my $timeout = 5;
5988
5989     my $devinfo = {};
5990
5991     my $rpcenv = PVE::RPCEnvironment::get();
5992
5993     my $conffile = PVE::QemuConfig->config_file($vmid);
5994     my $tmpfn = "$conffile.$$.tmp";
5995
5996     # Note: $oldconf is undef if VM does not exists
5997     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5998     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5999
6000     my %storage_limits;
6001
6002     my $print_devmap = sub {
6003         my $virtdev_hash = {};
6004
6005         my $cfgfn = "$tmpdir/qemu-server.conf";
6006
6007         # we can read the config - that is already extracted
6008         my $fh = IO::File->new($cfgfn, "r") ||
6009             "unable to read qemu-server.conf - $!\n";
6010
6011         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6012         if (-f $fwcfgfn) {
6013             my $pve_firewall_dir = '/etc/pve/firewall';
6014             mkdir $pve_firewall_dir; # make sure the dir exists
6015             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6016         }
6017
6018         while (defined(my $line = <$fh>)) {
6019             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6020                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6021                 die "archive does not contain data for drive '$virtdev'\n"
6022                     if !$devinfo->{$devname};
6023                 if (defined($opts->{storage})) {
6024                     $storeid = $opts->{storage} || 'local';
6025                 } elsif (!$storeid) {
6026                     $storeid = 'local';
6027                 }
6028                 $format = 'raw' if !$format;
6029                 $devinfo->{$devname}->{devname} = $devname;
6030                 $devinfo->{$devname}->{virtdev} = $virtdev;
6031                 $devinfo->{$devname}->{format} = $format;
6032                 $devinfo->{$devname}->{storeid} = $storeid;
6033
6034                 # check permission on storage
6035                 my $pool = $opts->{pool}; # todo: do we need that?
6036                 if ($user ne 'root@pam') {
6037                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6038                 }
6039
6040                 $storage_limits{$storeid} = $bwlimit;
6041
6042                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6043             }
6044         }
6045
6046         foreach my $key (keys %storage_limits) {
6047             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6048             next if !$limit;
6049             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6050             $storage_limits{$key} = $limit * 1024;
6051         }
6052
6053         foreach my $devname (keys %$devinfo) {
6054             die "found no device mapping information for device '$devname'\n"
6055                 if !$devinfo->{$devname}->{virtdev};
6056         }
6057
6058         # create empty/temp config
6059         if ($oldconf) {
6060             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6061             foreach_drive($oldconf, sub {
6062                 my ($ds, $drive) = @_;
6063
6064                 return if drive_is_cdrom($drive);
6065
6066                 my $volid = $drive->{file};
6067
6068                 return if !$volid || $volid =~ m|^/|;
6069
6070                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6071                 return if !$path || !$owner || ($owner != $vmid);
6072
6073                 # Note: only delete disk we want to restore
6074                 # other volumes will become unused
6075                 if ($virtdev_hash->{$ds}) {
6076                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6077                     if (my $err = $@) {
6078                         warn $err;
6079                     }
6080                 }
6081             });
6082
6083             # delete vmstate files
6084             # since after the restore we have no snapshots anymore
6085             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6086                 my $snap = $oldconf->{snapshots}->{$snapname};
6087                 if ($snap->{vmstate}) {
6088                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6089                     if (my $err = $@) {
6090                         warn $err;
6091                     }
6092                 }
6093             }
6094         }
6095
6096         my $map = {};
6097         foreach my $virtdev (sort keys %$virtdev_hash) {
6098             my $d = $virtdev_hash->{$virtdev};
6099             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6100             my $storeid = $d->{storeid};
6101             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6102
6103             my $map_opts = '';
6104             if (my $limit = $storage_limits{$storeid}) {
6105                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6106             }
6107
6108             # test if requested format is supported
6109             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6110             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6111             $d->{format} = $defFormat if !$supported;
6112
6113             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6114                                                   $d->{format}, undef, $alloc_size);
6115             print STDERR "new volume ID is '$volid'\n";
6116             $d->{volid} = $volid;
6117             my $path = PVE::Storage::path($cfg, $volid);
6118
6119             PVE::Storage::activate_volumes($cfg,[$volid]);
6120
6121             my $write_zeros = 1;
6122             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6123                 $write_zeros = 0;
6124             }
6125
6126             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6127
6128             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6129             $map->{$virtdev} = $volid;
6130         }
6131
6132         $fh->seek(0, 0) || die "seek failed - $!\n";
6133
6134         my $outfd = new IO::File ($tmpfn, "w") ||
6135             die "unable to write config for VM $vmid\n";
6136
6137         my $cookie = { netcount => 0 };
6138         while (defined(my $line = <$fh>)) {
6139             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6140         }
6141
6142         $fh->close();
6143         $outfd->close();
6144     };
6145
6146     eval {
6147         # enable interrupts
6148         local $SIG{INT} =
6149             local $SIG{TERM} =
6150             local $SIG{QUIT} =
6151             local $SIG{HUP} =
6152             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6153         local $SIG{ALRM} = sub { die "got timeout\n"; };
6154
6155         $oldtimeout = alarm($timeout);
6156
6157         my $parser = sub {
6158             my $line = shift;
6159
6160             print "$line\n";
6161
6162             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6163                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6164                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6165             } elsif ($line =~ m/^CTIME: /) {
6166                 # we correctly received the vma config, so we can disable
6167                 # the timeout now for disk allocation (set to 10 minutes, so
6168                 # that we always timeout if something goes wrong)
6169                 alarm(600);
6170                 &$print_devmap();
6171                 print $fifofh "done\n";
6172                 my $tmp = $oldtimeout || 0;
6173                 $oldtimeout = undef;
6174                 alarm($tmp);
6175                 close($fifofh);
6176             }
6177         };
6178
6179         print "restore vma archive: $dbg_cmdstring\n";
6180         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6181     };
6182     my $err = $@;
6183
6184     alarm($oldtimeout) if $oldtimeout;
6185
6186     my $vollist = [];
6187     foreach my $devname (keys %$devinfo) {
6188         my $volid = $devinfo->{$devname}->{volid};
6189         push @$vollist, $volid if $volid;
6190     }
6191
6192     PVE::Storage::deactivate_volumes($cfg, $vollist);
6193
6194     unlink $mapfifo;
6195
6196     if ($err) {
6197         rmtree $tmpdir;
6198         unlink $tmpfn;
6199
6200         foreach my $devname (keys %$devinfo) {
6201             my $volid = $devinfo->{$devname}->{volid};
6202             next if !$volid;
6203             eval {
6204                 if ($volid =~ m|^/|) {
6205                     unlink $volid || die 'unlink failed\n';
6206                 } else {
6207                     PVE::Storage::vdisk_free($cfg, $volid);
6208                 }
6209                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6210             };
6211             print STDERR "unable to cleanup '$volid' - $@" if $@;
6212         }
6213         die $err;
6214     }
6215
6216     rmtree $tmpdir;
6217
6218     rename($tmpfn, $conffile) ||
6219         die "unable to commit configuration file '$conffile'\n";
6220
6221     PVE::Cluster::cfs_update(); # make sure we read new file
6222
6223     eval { rescan($vmid, 1); };
6224     warn $@ if $@;
6225 }
6226
6227 sub restore_tar_archive {
6228     my ($archive, $vmid, $user, $opts) = @_;
6229
6230     if ($archive ne '-') {
6231         my $firstfile = tar_archive_read_firstfile($archive);
6232         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6233             if $firstfile ne 'qemu-server.conf';
6234     }
6235
6236     my $storecfg = PVE::Storage::config();
6237
6238     # destroy existing data - keep empty config
6239     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6240     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6241
6242     my $tocmd = "/usr/lib/qemu-server/qmextract";
6243
6244     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6245     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6246     $tocmd .= ' --prealloc' if $opts->{prealloc};
6247     $tocmd .= ' --info' if $opts->{info};
6248
6249     # tar option "xf" does not autodetect compression when read from STDIN,
6250     # so we pipe to zcat
6251     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6252         PVE::Tools::shellquote("--to-command=$tocmd");
6253
6254     my $tmpdir = "/var/tmp/vzdumptmp$$";
6255     mkpath $tmpdir;
6256
6257     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6258     local $ENV{VZDUMP_VMID} = $vmid;
6259     local $ENV{VZDUMP_USER} = $user;
6260
6261     my $conffile = PVE::QemuConfig->config_file($vmid);
6262     my $tmpfn = "$conffile.$$.tmp";
6263
6264     # disable interrupts (always do cleanups)
6265     local $SIG{INT} =
6266         local $SIG{TERM} =
6267         local $SIG{QUIT} =
6268         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6269
6270     eval {
6271         # enable interrupts
6272         local $SIG{INT} =
6273             local $SIG{TERM} =
6274             local $SIG{QUIT} =
6275             local $SIG{HUP} =
6276             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6277
6278         if ($archive eq '-') {
6279             print "extracting archive from STDIN\n";
6280             run_command($cmd, input => "<&STDIN");
6281         } else {
6282             print "extracting archive '$archive'\n";
6283             run_command($cmd);
6284         }
6285
6286         return if $opts->{info};
6287
6288         # read new mapping
6289         my $map = {};
6290         my $statfile = "$tmpdir/qmrestore.stat";
6291         if (my $fd = IO::File->new($statfile, "r")) {
6292             while (defined (my $line = <$fd>)) {
6293                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6294                     $map->{$1} = $2 if $1;
6295                 } else {
6296                     print STDERR "unable to parse line in statfile - $line\n";
6297                 }
6298             }
6299             $fd->close();
6300         }
6301
6302         my $confsrc = "$tmpdir/qemu-server.conf";
6303
6304         my $srcfd = new IO::File($confsrc, "r") ||
6305             die "unable to open file '$confsrc'\n";
6306
6307         my $outfd = new IO::File ($tmpfn, "w") ||
6308             die "unable to write config for VM $vmid\n";
6309
6310         my $cookie = { netcount => 0 };
6311         while (defined (my $line = <$srcfd>)) {
6312             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6313         }
6314
6315         $srcfd->close();
6316         $outfd->close();
6317     };
6318     my $err = $@;
6319
6320     if ($err) {
6321
6322         unlink $tmpfn;
6323
6324         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6325
6326         die $err;
6327     }
6328
6329     rmtree $tmpdir;
6330
6331     rename $tmpfn, $conffile ||
6332         die "unable to commit configuration file '$conffile'\n";
6333
6334     PVE::Cluster::cfs_update(); # make sure we read new file
6335
6336     eval { rescan($vmid, 1); };
6337     warn $@ if $@;
6338 };
6339
6340 sub foreach_storage_used_by_vm {
6341     my ($conf, $func) = @_;
6342
6343     my $sidhash = {};
6344
6345     foreach_drive($conf, sub {
6346         my ($ds, $drive) = @_;
6347         return if drive_is_cdrom($drive);
6348
6349         my $volid = $drive->{file};
6350
6351         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6352         $sidhash->{$sid} = $sid if $sid;
6353     });
6354
6355     foreach my $sid (sort keys %$sidhash) {
6356         &$func($sid);
6357     }
6358 }
6359
6360 sub do_snapshots_with_qemu {
6361     my ($storecfg, $volid) = @_;
6362
6363     my $storage_name = PVE::Storage::parse_volume_id($volid);
6364
6365     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6366         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6367         return 1;
6368     }
6369
6370     if ($volid =~ m/\.(qcow2|qed)$/){
6371         return 1;
6372     }
6373
6374     return undef;
6375 }
6376
6377 sub qga_check_running {
6378     my ($vmid, $nowarn) = @_;
6379
6380     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6381     if ($@) {
6382         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6383         return 0;
6384     }
6385     return 1;
6386 }
6387
6388 sub template_create {
6389     my ($vmid, $conf, $disk) = @_;
6390
6391     my $storecfg = PVE::Storage::config();
6392
6393     foreach_drive($conf, sub {
6394         my ($ds, $drive) = @_;
6395
6396         return if drive_is_cdrom($drive);
6397         return if $disk && $ds ne $disk;
6398
6399         my $volid = $drive->{file};
6400         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6401
6402         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6403         $drive->{file} = $voliddst;
6404         $conf->{$ds} = print_drive($vmid, $drive);
6405         PVE::QemuConfig->write_config($vmid, $conf);
6406     });
6407 }
6408
6409 sub qemu_img_convert {
6410     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6411
6412     my $storecfg = PVE::Storage::config();
6413     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6414     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6415
6416     if ($src_storeid && $dst_storeid) {
6417
6418         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6419
6420         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6421         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6422
6423         my $src_format = qemu_img_format($src_scfg, $src_volname);
6424         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6425
6426         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6427         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6428
6429         my $cmd = [];
6430         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6431         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6432         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6433         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6434         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6435         if ($is_zero_initialized) {
6436             push @$cmd, "zeroinit:$dst_path";
6437         } else {
6438             push @$cmd, $dst_path;
6439         }
6440
6441         my $parser = sub {
6442             my $line = shift;
6443             if($line =~ m/\((\S+)\/100\%\)/){
6444                 my $percent = $1;
6445                 my $transferred = int($size * $percent / 100);
6446                 my $remaining = $size - $transferred;
6447
6448                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6449             }
6450
6451         };
6452
6453         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6454         my $err = $@;
6455         die "copy failed: $err" if $err;
6456     }
6457 }
6458
6459 sub qemu_img_format {
6460     my ($scfg, $volname) = @_;
6461
6462     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6463         return $1;
6464     } else {
6465         return "raw";
6466     }
6467 }
6468
6469 sub qemu_drive_mirror {
6470     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6471
6472     $jobs = {} if !$jobs;
6473
6474     my $qemu_target;
6475     my $format;
6476     $jobs->{"drive-$drive"} = {};
6477
6478     if ($dst_volid =~ /^nbd:/) {
6479         $qemu_target = $dst_volid;
6480         $format = "nbd";
6481     } else {
6482         my $storecfg = PVE::Storage::config();
6483         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6484
6485         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6486
6487         $format = qemu_img_format($dst_scfg, $dst_volname);
6488
6489         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6490
6491         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6492     }
6493
6494     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6495     $opts->{format} = $format if $format;
6496
6497     print "drive mirror is starting for drive-$drive\n";
6498
6499     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6500
6501     if (my $err = $@) {
6502         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6503         die "mirroring error: $err";
6504     }
6505
6506     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6507 }
6508
6509 sub qemu_drive_mirror_monitor {
6510     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6511
6512     eval {
6513         my $err_complete = 0;
6514
6515         while (1) {
6516             die "storage migration timed out\n" if $err_complete > 300;
6517
6518             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6519
6520             my $running_mirror_jobs = {};
6521             foreach my $stat (@$stats) {
6522                 next if $stat->{type} ne 'mirror';
6523                 $running_mirror_jobs->{$stat->{device}} = $stat;
6524             }
6525
6526             my $readycounter = 0;
6527
6528             foreach my $job (keys %$jobs) {
6529
6530                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6531                     print "$job : finished\n";
6532                     delete $jobs->{$job};
6533                     next;
6534                 }
6535
6536                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6537
6538                 my $busy = $running_mirror_jobs->{$job}->{busy};
6539                 my $ready = $running_mirror_jobs->{$job}->{ready};
6540                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6541                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6542                     my $remaining = $total - $transferred;
6543                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6544
6545                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6546                 }
6547
6548                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6549             }
6550
6551             last if scalar(keys %$jobs) == 0;
6552
6553             if ($readycounter == scalar(keys %$jobs)) {
6554                 print "all mirroring jobs are ready \n";
6555                 last if $skipcomplete; #do the complete later
6556
6557                 if ($vmiddst && $vmiddst != $vmid) {
6558                     my $agent_running = $qga && qga_check_running($vmid);
6559                     if ($agent_running) {
6560                         print "freeze filesystem\n";
6561                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6562                     } else {
6563                         print "suspend vm\n";
6564                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6565                     }
6566
6567                     # if we clone a disk for a new target vm, we don't switch the disk
6568                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6569
6570                     if ($agent_running) {
6571                         print "unfreeze filesystem\n";
6572                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6573                     } else {
6574                         print "resume vm\n";
6575                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6576                     }
6577
6578                     last;
6579                 } else {
6580
6581                     foreach my $job (keys %$jobs) {
6582                         # try to switch the disk if source and destination are on the same guest
6583                         print "$job: Completing block job...\n";
6584
6585                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6586                         if ($@ =~ m/cannot be completed/) {
6587                             print "$job: Block job cannot be completed, try again.\n";
6588                             $err_complete++;
6589                         }else {
6590                             print "$job: Completed successfully.\n";
6591                             $jobs->{$job}->{complete} = 1;
6592                         }
6593                     }
6594                 }
6595             }
6596             sleep 1;
6597         }
6598     };
6599     my $err = $@;
6600
6601     if ($err) {
6602         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6603         die "mirroring error: $err";
6604     }
6605
6606 }
6607
6608 sub qemu_blockjobs_cancel {
6609     my ($vmid, $jobs) = @_;
6610
6611     foreach my $job (keys %$jobs) {
6612         print "$job: Cancelling block job\n";
6613         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6614         $jobs->{$job}->{cancel} = 1;
6615     }
6616
6617     while (1) {
6618         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6619
6620         my $running_jobs = {};
6621         foreach my $stat (@$stats) {
6622             $running_jobs->{$stat->{device}} = $stat;
6623         }
6624
6625         foreach my $job (keys %$jobs) {
6626
6627             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6628                 print "$job: Done.\n";
6629                 delete $jobs->{$job};
6630             }
6631         }
6632
6633         last if scalar(keys %$jobs) == 0;
6634
6635         sleep 1;
6636     }
6637 }
6638
6639 sub clone_disk {
6640     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6641         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6642
6643     my $newvolid;
6644
6645     if (!$full) {
6646         print "create linked clone of drive $drivename ($drive->{file})\n";
6647         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6648         push @$newvollist, $newvolid;
6649     } else {
6650
6651         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6652         $storeid = $storage if $storage;
6653
6654         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6655         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6656
6657         print "create full clone of drive $drivename ($drive->{file})\n";
6658         my $name = undef;
6659         if (drive_is_cloudinit($drive)) {
6660             $name = "vm-$newvmid-cloudinit";
6661             # cloudinit only supports raw and qcow2 atm:
6662             if ($dst_format eq 'qcow2') {
6663                 $name .= '.qcow2';
6664             } elsif ($dst_format ne 'raw') {
6665                 die "clone: unhandled format for cloudinit image\n";
6666             }
6667         }
6668         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6669         push @$newvollist, $newvolid;
6670
6671         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6672
6673         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6674         if (!$running || $snapname) {
6675             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6676         } else {
6677
6678             my $kvmver = get_running_qemu_version ($vmid);
6679             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6680                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6681                     if $drive->{iothread};
6682             }
6683
6684             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6685         }
6686     }
6687
6688     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6689
6690     my $disk = $drive;
6691     $disk->{format} = undef;
6692     $disk->{file} = $newvolid;
6693     $disk->{size} = $size;
6694
6695     return $disk;
6696 }
6697
6698 # this only works if VM is running
6699 sub get_current_qemu_machine {
6700     my ($vmid) = @_;
6701
6702     my $cmd = { execute => 'query-machines', arguments => {} };
6703     my $res = vm_qmp_command($vmid, $cmd);
6704
6705     my ($current, $default);
6706     foreach my $e (@$res) {
6707         $default = $e->{name} if $e->{'is-default'};
6708         $current = $e->{name} if $e->{'is-current'};
6709     }
6710
6711     # fallback to the default machine if current is not supported by qemu
6712     return $current || $default || 'pc';
6713 }
6714
6715 sub get_running_qemu_version {
6716     my ($vmid) = @_;
6717     my $cmd = { execute => 'query-version', arguments => {} };
6718     my $res = vm_qmp_command($vmid, $cmd);
6719     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6720 }
6721
6722 sub qemu_machine_feature_enabled {
6723     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6724
6725     my $current_major;
6726     my $current_minor;
6727
6728     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6729
6730         $current_major = $3;
6731         $current_minor = $4;
6732
6733     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6734
6735         $current_major = $1;
6736         $current_minor = $2;
6737     }
6738
6739     return 1 if $current_major > $version_major ||
6740                 ($current_major == $version_major &&
6741                  $current_minor >= $version_minor);
6742 }
6743
6744 sub qemu_machine_pxe {
6745     my ($vmid, $conf, $machine) = @_;
6746
6747     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6748
6749     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6750         $machine .= '.pxe';
6751     }
6752
6753     return $machine;
6754 }
6755
6756 sub qemu_use_old_bios_files {
6757     my ($machine_type) = @_;
6758
6759     return if !$machine_type;
6760
6761     my $use_old_bios_files = undef;
6762
6763     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6764         $machine_type = $1;
6765         $use_old_bios_files = 1;
6766     } else {
6767         my $kvmver = kvm_user_version();
6768         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6769         # load new efi bios files on migration. So this hack is required to allow
6770         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6771         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6772         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6773     }
6774
6775     return ($use_old_bios_files, $machine_type);
6776 }
6777
6778 sub create_efidisk {
6779     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6780
6781     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6782
6783     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6784     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6785     PVE::Storage::activate_volumes($storecfg, [$volid]);
6786
6787     my $path = PVE::Storage::path($storecfg, $volid);
6788     eval {
6789         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6790     };
6791     die "Copying EFI vars image failed: $@" if $@;
6792
6793     return ($volid, $vars_size);
6794 }
6795
6796 sub lspci {
6797
6798     my $devices = {};
6799
6800     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6801             my (undef, $id, $function) = @_;
6802             my $res = { id => $id, function => $function};
6803             push @{$devices->{$id}}, $res;
6804     });
6805
6806     # Entries should be sorted by functions.
6807     foreach my $id (keys %$devices) {
6808         my $dev = $devices->{$id};
6809         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6810     }
6811
6812     return $devices;
6813 }
6814
6815 sub vm_iothreads_list {
6816     my ($vmid) = @_;
6817
6818     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6819
6820     my $iothreads = {};
6821     foreach my $iothread (@$res) {
6822         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6823     }
6824
6825     return $iothreads;
6826 }
6827
6828 sub scsihw_infos {
6829     my ($conf, $drive) = @_;
6830
6831     my $maxdev = 0;
6832
6833     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6834         $maxdev = 7;
6835     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6836         $maxdev = 1;
6837     } else {
6838         $maxdev = 256;
6839     }
6840
6841     my $controller = int($drive->{index} / $maxdev);
6842     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6843
6844     return ($maxdev, $controller, $controller_prefix);
6845 }
6846
6847 sub add_hyperv_enlightenments {
6848     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6849
6850     return if $winversion < 6;
6851     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6852
6853     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6854
6855     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6856         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6857         push @$cpuFlags , 'hv_vapic';
6858         push @$cpuFlags , 'hv_time';
6859     } else {
6860         push @$cpuFlags , 'hv_spinlocks=0xffff';
6861     }
6862
6863     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6864         push @$cpuFlags , 'hv_reset';
6865         push @$cpuFlags , 'hv_vpindex';
6866         push @$cpuFlags , 'hv_runtime';
6867     }
6868
6869     if ($winversion >= 7) {
6870         push @$cpuFlags , 'hv_relaxed';
6871
6872         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6873             push @$cpuFlags , 'hv_synic';
6874             push @$cpuFlags , 'hv_stimer';
6875         }
6876     }
6877 }
6878
6879 sub windows_version {
6880     my ($ostype) = @_;
6881
6882     return 0 if !$ostype;
6883
6884     my $winversion = 0;
6885
6886     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6887         $winversion = 5;
6888     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6889         $winversion = 6;
6890     } elsif ($ostype =~ m/^win(\d+)$/) {
6891         $winversion = $1;
6892     }
6893
6894     return $winversion;
6895 }
6896
6897 sub resolve_dst_disk_format {
6898         my ($storecfg, $storeid, $src_volname, $format) = @_;
6899         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6900
6901         if (!$format) {
6902             # if no target format is specified, use the source disk format as hint
6903             if ($src_volname) {
6904                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6905                 $format = qemu_img_format($scfg, $src_volname);
6906             } else {
6907                 return $defFormat;
6908             }
6909         }
6910
6911         # test if requested format is supported - else use default
6912         my $supported = grep { $_ eq $format } @$validFormats;
6913         $format = $defFormat if !$supported;
6914         return $format;
6915 }
6916
6917 sub resolve_first_disk {
6918     my $conf = shift;
6919     my @disks = PVE::QemuServer::valid_drive_names();
6920     my $firstdisk;
6921     foreach my $ds (reverse @disks) {
6922         next if !$conf->{$ds};
6923         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6924         next if PVE::QemuServer::drive_is_cdrom($disk);
6925         $firstdisk = $ds;
6926     }
6927     return $firstdisk;
6928 }
6929
6930 sub generate_uuid {
6931     my ($uuid, $uuid_str);
6932     UUID::generate($uuid);
6933     UUID::unparse($uuid, $uuid_str);
6934     return $uuid_str;
6935 }
6936
6937 sub generate_smbios1_uuid {
6938     return "uuid=".generate_uuid();
6939 }
6940
6941 sub nbd_stop {
6942     my ($vmid) = @_;
6943
6944     vm_mon_cmd($vmid, 'nbd-server-stop');
6945 }
6946
6947 # bash completion helper
6948
6949 sub complete_backup_archives {
6950     my ($cmdname, $pname, $cvalue) = @_;
6951
6952     my $cfg = PVE::Storage::config();
6953
6954     my $storeid;
6955
6956     if ($cvalue =~ m/^([^:]+):/) {
6957         $storeid = $1;
6958     }
6959
6960     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6961
6962     my $res = [];
6963     foreach my $id (keys %$data) {
6964         foreach my $item (@{$data->{$id}}) {
6965             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6966             push @$res, $item->{volid} if defined($item->{volid});
6967         }
6968     }
6969
6970     return $res;
6971 }
6972
6973 my $complete_vmid_full = sub {
6974     my ($running) = @_;
6975
6976     my $idlist = vmstatus();
6977
6978     my $res = [];
6979
6980     foreach my $id (keys %$idlist) {
6981         my $d = $idlist->{$id};
6982         if (defined($running)) {
6983             next if $d->{template};
6984             next if $running && $d->{status} ne 'running';
6985             next if !$running && $d->{status} eq 'running';
6986         }
6987         push @$res, $id;
6988
6989     }
6990     return $res;
6991 };
6992
6993 sub complete_vmid {
6994     return &$complete_vmid_full();
6995 }
6996
6997 sub complete_vmid_stopped {
6998     return &$complete_vmid_full(0);
6999 }
7000
7001 sub complete_vmid_running {
7002     return &$complete_vmid_full(1);
7003 }
7004
7005 sub complete_storage {
7006
7007     my $cfg = PVE::Storage::config();
7008     my $ids = $cfg->{ids};
7009
7010     my $res = [];
7011     foreach my $sid (keys %$ids) {
7012         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7013         next if !$ids->{$sid}->{content}->{images};
7014         push @$res, $sid;
7015     }
7016
7017     return $res;
7018 }
7019
7020 1;