PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149     'Cascadelake-Server' => 'GenuineIntel',
 150     KnightsMill => 'GenuineIntel',
 151
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my @supported_cpu_flags = (
 174     'pcid',
 175     'spec-ctrl',
 176     'ibpb',
 177     'ssbd',
 178     'virt-ssbd',
 179     'amd-ssbd',
 180     'amd-no-ssb',
 181     'pdpe1gb',
 182     'md-clear',
 183     'hv-tlbflush',
 184     'hv-evmcs',
 185     'aes'
 186 );
 187 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 188
 189 my $cpu_fmt = {
 190     cputype => {
 191         description => "Emulated CPU type.",
 192         type => 'string',
 193         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 194         default => 'kvm64',
 195         default_key => 1,
 196     },
 197     hidden => {
 198         description => "Do not identify as a KVM virtual machine.",
 199         type => 'boolean',
 200         optional => 1,
 201         default => 0
 202     },
 203     'hv-vendor-id' => {
 204         type => 'string',
 205         pattern => qr/[a-zA-Z0-9]{1,12}/,
 206         format_description => 'vendor-id',
 207         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 208         optional => 1,
 209     },
 210     flags => {
 211         description => "List of additional CPU flags separated by ';'."
 212                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 213                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 214         format_description => '+FLAG[;-FLAG...]',
 215         type => 'string',
 216         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 217         optional => 1,
 218     },
 219 };
 220
 221 my $watchdog_fmt = {
 222     model => {
 223         default_key => 1,
 224         type => 'string',
 225         enum => [qw(i6300esb ib700)],
 226         description => "Watchdog type to emulate.",
 227         default => 'i6300esb',
 228         optional => 1,
 229     },
 230     action => {
 231         type => 'string',
 232         enum => [qw(reset shutdown poweroff pause debug none)],
 233         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 234         optional => 1,
 235     },
 236 };
 237 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 238
 239 my $agent_fmt = {
 240     enabled => {
 241         description => "Enable/disable Qemu GuestAgent.",
 242         type => 'boolean',
 243         default => 0,
 244         default_key => 1,
 245     },
 246     fstrim_cloned_disks => {
 247         description => "Run fstrim after cloning/moving a disk.",
 248         type => 'boolean',
 249         optional => 1,
 250         default => 0
 251     },
 252 };
 253
 254 my $vga_fmt = {
 255     type => {
 256         description => "Select the VGA type.",
 257         type => 'string',
 258         default => 'std',
 259         optional => 1,
 260         default_key => 1,
 261         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 262     },
 263     memory => {
 264         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 265         type => 'integer',
 266         optional => 1,
 267         minimum => 4,
 268         maximum => 512,
 269     },
 270 };
 271
 272 my $ivshmem_fmt = {
 273     size => {
 274         type => 'integer',
 275         minimum => 1,
 276         description => "The size of the file in MB.",
 277     },
 278     name => {
 279         type => 'string',
 280         pattern => '[a-zA-Z0-9\-]+',
 281         optional => 1,
 282         format_description => 'string',
 283         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 284     },
 285 };
 286
 287 my $audio_fmt = {
 288     device => {
 289         type => 'string',
 290         enum => [qw(ich9-intel-hda intel-hda AC97)],
 291         description =>  "Configure an audio device."
 292     },
 293     driver =>  {
 294         type => 'string',
 295         enum => ['spice'],
 296         default => 'spice',
 297         optional => 1,
 298         description => "Driver backend for the audio device."
 299     },
 300 };
 301
 302 my $spice_enhancements_fmt = {
 303     foldersharing => {
 304         type => 'boolean',
 305         optional => 1,
 306         default => '0',
 307         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 308     },
 309     videostreaming =>  {
 310         type => 'string',
 311         enum => ['off', 'all', 'filter'],
 312         default => 'off',
 313         optional => 1,
 314         description => "Enable video streaming. Uses compression for detected video streams."
 315     },
 316 };
 317
 318 my $confdesc = {
 319     onboot => {
 320         optional => 1,
 321         type => 'boolean',
 322         description => "Specifies whether a VM will be started during system bootup.",
 323         default => 0,
 324     },
 325     autostart => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Automatic restart after crash (currently ignored).",
 329         default => 0,
 330     },
 331     hotplug => {
 332         optional => 1,
 333         type => 'string', format => 'pve-hotplug-features',
 334         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 335         default => 'network,disk,usb',
 336     },
 337     reboot => {
 338         optional => 1,
 339         type => 'boolean',
 340         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 341         default => 1,
 342     },
 343     lock => {
 344         optional => 1,
 345         type => 'string',
 346         description => "Lock/unlock the VM.",
 347         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 348     },
 349     cpulimit => {
 350         optional => 1,
 351         type => 'number',
 352         description => "Limit of CPU usage.",
 353         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 354         minimum => 0,
 355         maximum => 128,
 356         default => 0,
 357     },
 358     cpuunits => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "CPU weight for a VM.",
 362         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 363         minimum => 2,
 364         maximum => 262144,
 365         default => 1024,
 366     },
 367     memory => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 371         minimum => 16,
 372         default => 512,
 373     },
 374     balloon => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 378         minimum => 0,
 379     },
 380     shares => {
 381         optional => 1,
 382         type => 'integer',
 383         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 384         minimum => 0,
 385         maximum => 50000,
 386         default => 1000,
 387     },
 388     keyboard => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 392                        "It should not be necessary to set it.",
 393         enum => PVE::Tools::kvmkeymaplist(),
 394         default => undef,
 395     },
 396     name => {
 397         optional => 1,
 398         type => 'string', format => 'dns-name',
 399         description => "Set a name for the VM. Only used on the configuration web interface.",
 400     },
 401     scsihw => {
 402         optional => 1,
 403         type => 'string',
 404         description => "SCSI controller model",
 405         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 406         default => 'lsi',
 407     },
 408     description => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 412     },
 413     ostype => {
 414         optional => 1,
 415         type => 'string',
 416         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 417         description => "Specify guest operating system.",
 418         verbose_description => <<EODESC,
 419 Specify guest operating system. This is used to enable special
 420 optimization/features for specific operating systems:
 421
 422 [horizontal]
 423 other;; unspecified OS
 424 wxp;; Microsoft Windows XP
 425 w2k;; Microsoft Windows 2000
 426 w2k3;; Microsoft Windows 2003
 427 w2k8;; Microsoft Windows 2008
 428 wvista;; Microsoft Windows Vista
 429 win7;; Microsoft Windows 7
 430 win8;; Microsoft Windows 8/2012/2012r2
 431 win10;; Microsoft Windows 10/2016
 432 l24;; Linux 2.4 Kernel
 433 l26;; Linux 2.6/3.X Kernel
 434 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 435 EODESC
 436     },
 437     boot => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 441         pattern => '[acdn]{1,4}',
 442         default => 'cdn',
 443     },
 444     bootdisk => {
 445         optional => 1,
 446         type => 'string', format => 'pve-qm-bootdisk',
 447         description => "Enable booting from specified disk.",
 448         pattern => '(ide|sata|scsi|virtio)\d+',
 449     },
 450     smp => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of CPUs. Please use option -sockets instead.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     sockets => {
 458         optional => 1,
 459         type => 'integer',
 460         description => "The number of CPU sockets.",
 461         minimum => 1,
 462         default => 1,
 463     },
 464     cores => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "The number of cores per socket.",
 468         minimum => 1,
 469         default => 1,
 470     },
 471     numa => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable NUMA.",
 475         default => 0,
 476     },
 477     hugepages => {
 478         optional => 1,
 479         type => 'string',
 480         description => "Enable/disable hugepages memory.",
 481         enum => [qw(any 2 1024)],
 482     },
 483     vcpus => {
 484         optional => 1,
 485         type => 'integer',
 486         description => "Number of hotplugged vcpus.",
 487         minimum => 1,
 488         default => 0,
 489     },
 490     acpi => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable ACPI.",
 494         default => 1,
 495     },
 496     agent => {
 497         optional => 1,
 498         description => "Enable/disable Qemu GuestAgent and its properties.",
 499         type => 'string',
 500         format => $agent_fmt,
 501     },
 502     kvm => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Enable/disable KVM hardware virtualization.",
 506         default => 1,
 507     },
 508     tdf => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Enable/disable time drift fix.",
 512         default => 0,
 513     },
 514     localtime => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 518     },
 519     freeze => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 523     },
 524     vga => {
 525         optional => 1,
 526         type => 'string', format => $vga_fmt,
 527         description => "Configure the VGA hardware.",
 528         verbose_description => "Configure the VGA Hardware. If you want to use ".
 529             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 530             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 531             "is 'std' for all OS types besides some Windows versions (XP and " .
 532             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 533             "display server. For win* OS you can select how many independent " .
 534             "displays you want, Linux guests can add displays them self.\n".
 535             "You can also run without any graphic card, using a serial device as terminal.",
 536     },
 537     watchdog => {
 538         optional => 1,
 539         type => 'string', format => 'pve-qm-watchdog',
 540         description => "Create a virtual hardware watchdog device.",
 541         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 542             " (by a guest action), the watchdog must be periodically polled " .
 543             "by an agent inside the guest or else the watchdog will reset " .
 544             "the guest (or execute the respective action specified)",
 545     },
 546     startdate => {
 547         optional => 1,
 548         type => 'string',
 549         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 550         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 551         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 552         default => 'now',
 553     },
 554     startup =>  get_standard_option('pve-startup-order'),
 555     template => {
 556         optional => 1,
 557         type => 'boolean',
 558         description => "Enable/disable Template.",
 559         default => 0,
 560     },
 561     args => {
 562         optional => 1,
 563         type => 'string',
 564         description => "Arbitrary arguments passed to kvm.",
 565         verbose_description => <<EODESCR,
 566 Arbitrary arguments passed to kvm, for example:
 567
 568 args: -no-reboot -no-hpet
 569
 570 NOTE: this option is for experts only.
 571 EODESCR
 572     },
 573     tablet => {
 574         optional => 1,
 575         type => 'boolean',
 576         default => 1,
 577         description => "Enable/disable the USB tablet device.",
 578         verbose_description => "Enable/disable the USB tablet device. This device is " .
 579             "usually needed to allow absolute mouse positioning with VNC. " .
 580             "Else the mouse runs out of sync with normal VNC clients. " .
 581             "If you're running lots of console-only guests on one host, " .
 582             "you may consider disabling this to save some context switches. " .
 583             "This is turned off by default if you use spice (-vga=qxl).",
 584     },
 585     migrate_speed => {
 586         optional => 1,
 587         type => 'integer',
 588         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 589         minimum => 0,
 590         default => 0,
 591     },
 592     migrate_downtime => {
 593         optional => 1,
 594         type => 'number',
 595         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 596         minimum => 0,
 597         default => 0.1,
 598     },
 599     cdrom => {
 600         optional => 1,
 601         type => 'string', format => 'pve-qm-ide',
 602         typetext => '<volume>',
 603         description => "This is an alias for option -ide2",
 604     },
 605     cpu => {
 606         optional => 1,
 607         description => "Emulated CPU type.",
 608         type => 'string',
 609         format => $cpu_fmt,
 610     },
 611     parent => get_standard_option('pve-snapshot-name', {
 612         optional => 1,
 613         description => "Parent snapshot name. This is used internally, and should not be modified.",
 614     }),
 615     snaptime => {
 616         optional => 1,
 617         description => "Timestamp for snapshots.",
 618         type => 'integer',
 619         minimum => 0,
 620     },
 621     vmstate => {
 622         optional => 1,
 623         type => 'string', format => 'pve-volume-id',
 624         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 625     },
 626     vmstatestorage => get_standard_option('pve-storage-id', {
 627         description => "Default storage for VM state volumes/files.",
 628         optional => 1,
 629     }),
 630     runningmachine => get_standard_option('pve-qemu-machine', {
 631         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 632     }),
 633     machine => get_standard_option('pve-qemu-machine'),
 634     arch => {
 635         description => "Virtual processor architecture. Defaults to the host.",
 636         optional => 1,
 637         type => 'string',
 638         enum => [qw(x86_64 aarch64)],
 639     },
 640     smbios1 => {
 641         description => "Specify SMBIOS type 1 fields.",
 642         type => 'string', format => 'pve-qm-smbios1',
 643         maxLength => 512,
 644         optional => 1,
 645     },
 646     protection => {
 647         optional => 1,
 648         type => 'boolean',
 649         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 650         default => 0,
 651     },
 652     bios => {
 653         optional => 1,
 654         type => 'string',
 655         enum => [ qw(seabios ovmf) ],
 656         description => "Select BIOS implementation.",
 657         default => 'seabios',
 658     },
 659     vmgenid => {
 660         type => 'string',
 661         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 662         format_description => 'UUID',
 663         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 664         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 665             " 128-bit integer value identifier to the guest OS. This allows to".
 666             " notify the guest operating system when the virtual machine is".
 667             " executed with a different configuration (e.g. snapshot execution".
 668             " or creation from a template). The guest operating system notices".
 669             " the change, and is then able to react as appropriate by marking".
 670             " its copies of distributed databases as dirty, re-initializing its".
 671             " random number generator, etc.\n".
 672             "Note that auto-creation only works when done throug API/CLI create".
 673             " or update methods, but not when manually editing the config file.",
 674         default => "1 (autogenerated)",
 675         optional => 1,
 676     },
 677     hookscript => {
 678         type => 'string',
 679         format => 'pve-volume-id',
 680         optional => 1,
 681         description => "Script that will be executed during various steps in the vms lifetime.",
 682     },
 683     ivshmem => {
 684         type => 'string',
 685         format => $ivshmem_fmt,
 686         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 687         optional => 1,
 688     },
 689     audio0 => {
 690         type => 'string',
 691         format => $audio_fmt,
 692         description => "Configure a audio device, useful in combination with QXL/Spice.",
 693         optional => 1
 694     },
 695     spice_enhancements => {
 696         type => 'string',
 697         format => $spice_enhancements_fmt,
 698         description => "Configure additional enhancements for SPICE.",
 699         optional => 1
 700     },
 701 };
 702
 703 my $cicustom_fmt = {
 704     meta => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711     network => {
 712         type => 'string',
 713         optional => 1,
 714         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 715         format => 'pve-volume-id',
 716         format_description => 'volume',
 717     },
 718     user => {
 719         type => 'string',
 720         optional => 1,
 721         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 722         format => 'pve-volume-id',
 723         format_description => 'volume',
 724     },
 725 };
 726 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 727
 728 my $confdesc_cloudinit = {
 729     citype => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 733         enum => ['configdrive2', 'nocloud'],
 734     },
 735     ciuser => {
 736         optional => 1,
 737         type => 'string',
 738         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 739     },
 740     cipassword => {
 741         optional => 1,
 742         type => 'string',
 743         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 744     },
 745     cicustom => {
 746         optional => 1,
 747         type => 'string',
 748         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 749         format => 'pve-qm-cicustom',
 750     },
 751     searchdomain => {
 752         optional => 1,
 753         type => 'string',
 754         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 755     },
 756     nameserver => {
 757         optional => 1,
 758         type => 'string', format => 'address-list',
 759         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 760     },
 761     sshkeys => {
 762         optional => 1,
 763         type => 'string',
 764         format => 'urlencoded',
 765         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 766     },
 767 };
 768
 769 # what about other qemu settings ?
 770 #cpu => 'string',
 771 #machine => 'string',
 772 #fda => 'file',
 773 #fdb => 'file',
 774 #mtdblock => 'file',
 775 #sd => 'file',
 776 #pflash => 'file',
 777 #snapshot => 'bool',
 778 #bootp => 'file',
 779 ##tftp => 'dir',
 780 ##smb => 'dir',
 781 #kernel => 'file',
 782 #append => 'string',
 783 #initrd => 'file',
 784 ##soundhw => 'string',
 785
 786 while (my ($k, $v) = each %$confdesc) {
 787     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 788 }
 789
 790 my $MAX_IDE_DISKS = 4;
 791 my $MAX_SCSI_DISKS = 14;
 792 my $MAX_VIRTIO_DISKS = 16;
 793 my $MAX_SATA_DISKS = 6;
 794 my $MAX_USB_DEVICES = 5;
 795 my $MAX_NETS = 32;
 796 my $MAX_UNUSED_DISKS = 256;
 797 my $MAX_HOSTPCI_DEVICES = 16;
 798 my $MAX_SERIAL_PORTS = 4;
 799 my $MAX_PARALLEL_PORTS = 3;
 800 my $MAX_NUMA = 8;
 801
 802 my $numa_fmt = {
 803     cpus => {
 804         type => "string",
 805         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 806         description => "CPUs accessing this NUMA node.",
 807         format_description => "id[-id];...",
 808     },
 809     memory => {
 810         type => "number",
 811         description => "Amount of memory this NUMA node provides.",
 812         optional => 1,
 813     },
 814     hostnodes => {
 815         type => "string",
 816         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 817         description => "Host NUMA nodes to use.",
 818         format_description => "id[-id];...",
 819         optional => 1,
 820     },
 821     policy => {
 822         type => 'string',
 823         enum => [qw(preferred bind interleave)],
 824         description => "NUMA allocation policy.",
 825         optional => 1,
 826     },
 827 };
 828 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 829 my $numadesc = {
 830     optional => 1,
 831     type => 'string', format => $numa_fmt,
 832     description => "NUMA topology.",
 833 };
 834 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 835
 836 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 837     $confdesc->{"numa$i"} = $numadesc;
 838 }
 839
 840 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 841                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 842                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 843 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 844
 845 my $net_fmt_bridge_descr = <<__EOD__;
 846 Bridge to attach the network device to. The Proxmox VE standard bridge
 847 is called 'vmbr0'.
 848
 849 If you do not specify a bridge, we create a kvm user (NATed) network
 850 device, which provides DHCP and DNS services. The following addresses
 851 are used:
 852
 853  10.0.2.2   Gateway
 854  10.0.2.3   DNS Server
 855  10.0.2.4   SMB Server
 856
 857 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 858 __EOD__
 859
 860 my $net_fmt = {
 861     macaddr  => get_standard_option('mac-addr', {
 862         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 863     }),
 864     model => {
 865         type => 'string',
 866         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 867         enum => $nic_model_list,
 868         default_key => 1,
 869     },
 870     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 871     bridge => {
 872         type => 'string',
 873         description => $net_fmt_bridge_descr,
 874         format_description => 'bridge',
 875         optional => 1,
 876     },
 877     queues => {
 878         type => 'integer',
 879         minimum => 0, maximum => 16,
 880         description => 'Number of packet queues to be used on the device.',
 881         optional => 1,
 882     },
 883     rate => {
 884         type => 'number',
 885         minimum => 0,
 886         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 887         optional => 1,
 888     },
 889     tag => {
 890         type => 'integer',
 891         minimum => 1, maximum => 4094,
 892         description => 'VLAN tag to apply to packets on this interface.',
 893         optional => 1,
 894     },
 895     trunks => {
 896         type => 'string',
 897         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 898         description => 'VLAN trunks to pass through this interface.',
 899         format_description => 'vlanid[;vlanid...]',
 900         optional => 1,
 901     },
 902     firewall => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be protected by the firewall.',
 905         optional => 1,
 906     },
 907     link_down => {
 908         type => 'boolean',
 909         description => 'Whether this interface should be disconnected (like pulling the plug).',
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 966 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 967
 968 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 969 EODESCR
 970 };
 971 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 972
 973 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 974     $confdesc->{"net$i"} = $netdesc;
 975     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 976 }
 977
 978 foreach my $key (keys %$confdesc_cloudinit) {
 979     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 980 }
 981
 982 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 983 sub verify_volume_id_or_qm_path {
 984     my ($volid, $noerr) = @_;
 985
 986     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 987         return $volid;
 988     }
 989
 990     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 991     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 992     if ($@) {
 993         return undef if $noerr;
 994         die $@;
 995     }
 996     return $volid;
 997 }
 998
 999 my $drivename_hash;
1000
1001 my %drivedesc_base = (
1002     volume => { alias => 'file' },
1003     file => {
1004         type => 'string',
1005         format => 'pve-volume-id-or-qm-path',
1006         default_key => 1,
1007         format_description => 'volume',
1008         description => "The drive's backing volume.",
1009     },
1010     media => {
1011         type => 'string',
1012         enum => [qw(cdrom disk)],
1013         description => "The drive's media type.",
1014         default => 'disk',
1015         optional => 1
1016     },
1017     cyls => {
1018         type => 'integer',
1019         description => "Force the drive's physical geometry to have a specific cylinder count.",
1020         optional => 1
1021     },
1022     heads => {
1023         type => 'integer',
1024         description => "Force the drive's physical geometry to have a specific head count.",
1025         optional => 1
1026     },
1027     secs => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific sector count.",
1030         optional => 1
1031     },
1032     trans => {
1033         type => 'string',
1034         enum => [qw(none lba auto)],
1035         description => "Force disk geometry bios translation mode.",
1036         optional => 1,
1037     },
1038     snapshot => {
1039         type => 'boolean',
1040         description => "Controls qemu's snapshot mode feature."
1041             . " If activated, changes made to the disk are temporary and will"
1042             . " be discarded when the VM is shutdown.",
1043         optional => 1,
1044     },
1045     cache => {
1046         type => 'string',
1047         enum => [qw(none writethrough writeback unsafe directsync)],
1048         description => "The drive's cache mode",
1049         optional => 1,
1050     },
1051     format => get_standard_option('pve-qm-image-format'),
1052     size => {
1053         type => 'string',
1054         format => 'disk-size',
1055         format_description => 'DiskSize',
1056         description => "Disk size. This is purely informational and has no effect.",
1057         optional => 1,
1058     },
1059     backup => {
1060         type => 'boolean',
1061         description => "Whether the drive should be included when making backups.",
1062         optional => 1,
1063     },
1064     replicate => {
1065         type => 'boolean',
1066         description => 'Whether the drive should considered for replication jobs.',
1067         optional => 1,
1068         default => 1,
1069     },
1070     rerror => {
1071         type => 'string',
1072         enum => [qw(ignore report stop)],
1073         description => 'Read error action.',
1074         optional => 1,
1075     },
1076     werror => {
1077         type => 'string',
1078         enum => [qw(enospc ignore report stop)],
1079         description => 'Write error action.',
1080         optional => 1,
1081     },
1082     aio => {
1083         type => 'string',
1084         enum => [qw(native threads)],
1085         description => 'AIO type to use.',
1086         optional => 1,
1087     },
1088     discard => {
1089         type => 'string',
1090         enum => [qw(ignore on)],
1091         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1092         optional => 1,
1093     },
1094     detect_zeroes => {
1095         type => 'boolean',
1096         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1097         optional => 1,
1098     },
1099     serial => {
1100         type => 'string',
1101         format => 'urlencoded',
1102         format_description => 'serial',
1103         maxLength => 20*3, # *3 since it's %xx url enoded
1104         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1105         optional => 1,
1106     },
1107     shared => {
1108         type => 'boolean',
1109         description => 'Mark this locally-managed volume as available on all nodes',
1110         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1111         optional => 1,
1112         default => 0,
1113     }
1114 );
1115
1116 my %iothread_fmt = ( iothread => {
1117         type => 'boolean',
1118         description => "Whether to use iothreads for this drive",
1119         optional => 1,
1120 });
1121
1122 my %model_fmt = (
1123     model => {
1124         type => 'string',
1125         format => 'urlencoded',
1126         format_description => 'model',
1127         maxLength => 40*3, # *3 since it's %xx url enoded
1128         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1129         optional => 1,
1130     },
1131 );
1132
1133 my %queues_fmt = (
1134     queues => {
1135         type => 'integer',
1136         description => "Number of queues.",
1137         minimum => 2,
1138         optional => 1
1139     }
1140 );
1141
1142 my %scsiblock_fmt = (
1143     scsiblock => {
1144         type => 'boolean',
1145         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1146         optional => 1,
1147         default => 0,
1148     },
1149 );
1150
1151 my %ssd_fmt = (
1152     ssd => {
1153         type => 'boolean',
1154         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1155         optional => 1,
1156     },
1157 );
1158
1159 my %wwn_fmt = (
1160     wwn => {
1161         type => 'string',
1162         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1163         format_description => 'wwn',
1164         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my $add_throttle_desc = sub {
1170     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1171     my $d = {
1172         type => $type,
1173         format_description => $unit,
1174         description => "Maximum $what in $longunit.",
1175         optional => 1,
1176     };
1177     $d->{minimum} = $minimum if defined($minimum);
1178     $drivedesc_base{$key} = $d;
1179 };
1180 # throughput: (leaky bucket)
1181 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1182 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1183 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1184 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1185 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1186 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1187 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1188 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1189 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1190
1191 # pools: (pool of IO before throttling starts taking effect)
1192 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1193 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1194 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1195 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1196 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1197 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1198
1199 # burst lengths
1200 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1201 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1202 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1204 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206
1207 # legacy support
1208 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1209 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1210 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1211 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1212
1213 my $ide_fmt = {
1214     %drivedesc_base,
1215     %model_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1220
1221 my $idedesc = {
1222     optional => 1,
1223     type => 'string', format => $ide_fmt,
1224     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1225 };
1226 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1227
1228 my $scsi_fmt = {
1229     %drivedesc_base,
1230     %iothread_fmt,
1231     %queues_fmt,
1232     %scsiblock_fmt,
1233     %ssd_fmt,
1234     %wwn_fmt,
1235 };
1236 my $scsidesc = {
1237     optional => 1,
1238     type => 'string', format => $scsi_fmt,
1239     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1240 };
1241 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1242
1243 my $sata_fmt = {
1244     %drivedesc_base,
1245     %ssd_fmt,
1246     %wwn_fmt,
1247 };
1248 my $satadesc = {
1249     optional => 1,
1250     type => 'string', format => $sata_fmt,
1251     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1252 };
1253 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1254
1255 my $virtio_fmt = {
1256     %drivedesc_base,
1257     %iothread_fmt,
1258 };
1259 my $virtiodesc = {
1260     optional => 1,
1261     type => 'string', format => $virtio_fmt,
1262     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1265
1266 my $alldrive_fmt = {
1267     %drivedesc_base,
1268     %iothread_fmt,
1269     %model_fmt,
1270     %queues_fmt,
1271     %scsiblock_fmt,
1272     %ssd_fmt,
1273     %wwn_fmt,
1274 };
1275
1276 my $efidisk_fmt = {
1277     volume => { alias => 'file' },
1278     file => {
1279         type => 'string',
1280         format => 'pve-volume-id-or-qm-path',
1281         default_key => 1,
1282         format_description => 'volume',
1283         description => "The drive's backing volume.",
1284     },
1285     format => get_standard_option('pve-qm-image-format'),
1286     size => {
1287         type => 'string',
1288         format => 'disk-size',
1289         format_description => 'DiskSize',
1290         description => "Disk size. This is purely informational and has no effect.",
1291         optional => 1,
1292     },
1293 };
1294
1295 my $efidisk_desc = {
1296     optional => 1,
1297     type => 'string', format => $efidisk_fmt,
1298     description => "Configure a Disk for storing EFI vars",
1299 };
1300
1301 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1302
1303 my $usb_fmt = {
1304     host => {
1305         default_key => 1,
1306         type => 'string', format => 'pve-qm-usb-device',
1307         format_description => 'HOSTUSBDEVICE|spice',
1308         description => <<EODESCR,
1309 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1310
1311  'bus-port(.port)*' (decimal numbers) or
1312  'vendor_id:product_id' (hexadeciaml numbers) or
1313  'spice'
1314
1315 You can use the 'lsusb -t' command to list existing usb devices.
1316
1317 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1318
1319 The value 'spice' can be used to add a usb redirection devices for spice.
1320 EODESCR
1321     },
1322     usb3 => {
1323         optional => 1,
1324         type => 'boolean',
1325         description => "Specifies whether if given host option is a USB3 device or port.",
1326         default => 0,
1327     },
1328 };
1329
1330 my $usbdesc = {
1331     optional => 1,
1332     type => 'string', format => $usb_fmt,
1333     description => "Configure an USB device (n is 0 to 4).",
1334 };
1335 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1336
1337 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1338 my $hostpci_fmt = {
1339     host => {
1340         default_key => 1,
1341         type => 'string',
1342         pattern => qr/$PCIRE(;$PCIRE)*/,
1343         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1344         description => <<EODESCR,
1345 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1346 of PCI virtual functions of the host. HOSTPCIID syntax is:
1347
1348 'bus:dev.func' (hexadecimal numbers)
1349
1350 You can us the 'lspci' command to list existing PCI devices.
1351 EODESCR
1352     },
1353     rombar => {
1354         type => 'boolean',
1355         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1356         optional => 1,
1357         default => 1,
1358     },
1359     romfile => {
1360         type => 'string',
1361         pattern => '[^,;]+',
1362         format_description => 'string',
1363         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1364         optional => 1,
1365     },
1366     pcie => {
1367         type => 'boolean',
1368         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1369         optional => 1,
1370         default => 0,
1371     },
1372     'x-vga' => {
1373         type => 'boolean',
1374         description =>  "Enable vfio-vga device support.",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'mdev' => {
1379         type => 'string',
1380         format_description => 'string',
1381         pattern => '[^/\.:]+',
1382         optional => 1,
1383         description => <<EODESCR
1384 The type of mediated device to use.
1385 An instance of this type will be created on startup of the VM and
1386 will be cleaned up when the VM stops.
1387 EODESCR
1388     }
1389 };
1390 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1391
1392 my $hostpcidesc = {
1393         optional => 1,
1394         type => 'string', format => 'pve-qm-hostpci',
1395         description => "Map host PCI devices into guest.",
1396         verbose_description =>  <<EODESCR,
1397 Map host PCI devices into guest.
1398
1399 NOTE: This option allows direct access to host hardware. So it is no longer
1400 possible to migrate such machines - use with special care.
1401
1402 CAUTION: Experimental! User reported problems with this option.
1403 EODESCR
1404 };
1405 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1406
1407 my $serialdesc = {
1408         optional => 1,
1409         type => 'string',
1410         pattern => '(/dev/.+|socket)',
1411         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1412         verbose_description =>  <<EODESCR,
1413 Create a serial device inside the VM (n is 0 to 3), and pass through a
1414 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1415 host side (use 'qm terminal' to open a terminal connection).
1416
1417 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1418
1419 CAUTION: Experimental! User reported problems with this option.
1420 EODESCR
1421 };
1422
1423 my $paralleldesc= {
1424         optional => 1,
1425         type => 'string',
1426         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1427         description =>  "Map host parallel devices (n is 0 to 2).",
1428         verbose_description =>  <<EODESCR,
1429 Map host parallel devices (n is 0 to 2).
1430
1431 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1432
1433 CAUTION: Experimental! User reported problems with this option.
1434 EODESCR
1435 };
1436
1437 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1438     $confdesc->{"parallel$i"} = $paralleldesc;
1439 }
1440
1441 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1442     $confdesc->{"serial$i"} = $serialdesc;
1443 }
1444
1445 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1446     $confdesc->{"hostpci$i"} = $hostpcidesc;
1447 }
1448
1449 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1450     $drivename_hash->{"ide$i"} = 1;
1451     $confdesc->{"ide$i"} = $idedesc;
1452 }
1453
1454 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1455     $drivename_hash->{"sata$i"} = 1;
1456     $confdesc->{"sata$i"} = $satadesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1460     $drivename_hash->{"scsi$i"} = 1;
1461     $confdesc->{"scsi$i"} = $scsidesc ;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1465     $drivename_hash->{"virtio$i"} = 1;
1466     $confdesc->{"virtio$i"} = $virtiodesc;
1467 }
1468
1469 $drivename_hash->{efidisk0} = 1;
1470 $confdesc->{efidisk0} = $efidisk_desc;
1471
1472 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1473     $confdesc->{"usb$i"} = $usbdesc;
1474 }
1475
1476 my $unuseddesc = {
1477     optional => 1,
1478     type => 'string', format => 'pve-volume-id',
1479     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1480 };
1481
1482 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1483     $confdesc->{"unused$i"} = $unuseddesc;
1484 }
1485
1486 my $kvm_api_version = 0;
1487
1488 sub kvm_version {
1489     return $kvm_api_version if $kvm_api_version;
1490
1491     open my $fh, '<', '/dev/kvm'
1492         or return undef;
1493
1494     # 0xae00 => KVM_GET_API_VERSION
1495     $kvm_api_version = ioctl($fh, 0xae00, 0);
1496
1497     return $kvm_api_version;
1498 }
1499
1500 my $kvm_user_version = {};
1501 my $kvm_mtime = {};
1502
1503 sub kvm_user_version {
1504     my ($binary) = @_;
1505
1506     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1507     my $st = stat($binary);
1508
1509     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1510     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1511         $cachedmtime == $st->mtime;
1512
1513     $kvm_user_version->{$binary} = 'unknown';
1514     $kvm_mtime->{$binary} = $st->mtime;
1515
1516     my $code = sub {
1517         my $line = shift;
1518         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1519             $kvm_user_version->{$binary} = $2;
1520         }
1521     };
1522
1523     eval { run_command([$binary, '--version'], outfunc => $code); };
1524     warn $@ if $@;
1525
1526     return $kvm_user_version->{$binary};
1527
1528 }
1529
1530 sub kernel_has_vhost_net {
1531     return -c '/dev/vhost-net';
1532 }
1533
1534 sub valid_drive_names {
1535     # order is important - used to autoselect boot disk
1536     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1537             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1538             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1539             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1540             'efidisk0');
1541 }
1542
1543 sub is_valid_drivename {
1544     my $dev = shift;
1545
1546     return defined($drivename_hash->{$dev});
1547 }
1548
1549 sub option_exists {
1550     my $key = shift;
1551     return defined($confdesc->{$key});
1552 }
1553
1554 sub nic_models {
1555     return $nic_model_list;
1556 }
1557
1558 sub os_list_description {
1559
1560     return {
1561         other => 'Other',
1562         wxp => 'Windows XP',
1563         w2k => 'Windows 2000',
1564         w2k3 =>, 'Windows 2003',
1565         w2k8 => 'Windows 2008',
1566         wvista => 'Windows Vista',
1567         win7 => 'Windows 7',
1568         win8 => 'Windows 8/2012',
1569         win10 => 'Windows 10/2016',
1570         l24 => 'Linux 2.4',
1571         l26 => 'Linux 2.6',
1572     };
1573 }
1574
1575 my $cdrom_path;
1576
1577 sub get_cdrom_path {
1578
1579     return  $cdrom_path if $cdrom_path;
1580
1581     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1582     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1583     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1584 }
1585
1586 sub get_iso_path {
1587     my ($storecfg, $vmid, $cdrom) = @_;
1588
1589     if ($cdrom eq 'cdrom') {
1590         return get_cdrom_path();
1591     } elsif ($cdrom eq 'none') {
1592         return '';
1593     } elsif ($cdrom =~ m|^/|) {
1594         return $cdrom;
1595     } else {
1596         return PVE::Storage::path($storecfg, $cdrom);
1597     }
1598 }
1599
1600 # try to convert old style file names to volume IDs
1601 sub filename_to_volume_id {
1602     my ($vmid, $file, $media) = @_;
1603
1604      if (!($file eq 'none' || $file eq 'cdrom' ||
1605           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1606
1607         return undef if $file =~ m|/|;
1608
1609         if ($media && $media eq 'cdrom') {
1610             $file = "local:iso/$file";
1611         } else {
1612             $file = "local:$vmid/$file";
1613         }
1614     }
1615
1616     return $file;
1617 }
1618
1619 sub verify_media_type {
1620     my ($opt, $vtype, $media) = @_;
1621
1622     return if !$media;
1623
1624     my $etype;
1625     if ($media eq 'disk') {
1626         $etype = 'images';
1627     } elsif ($media eq 'cdrom') {
1628         $etype = 'iso';
1629     } else {
1630         die "internal error";
1631     }
1632
1633     return if ($vtype eq $etype);
1634
1635     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1636 }
1637
1638 sub cleanup_drive_path {
1639     my ($opt, $storecfg, $drive) = @_;
1640
1641     # try to convert filesystem paths to volume IDs
1642
1643     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1644         ($drive->{file} !~ m|^/dev/.+|) &&
1645         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1646         ($drive->{file} !~ m/^\d+$/)) {
1647         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1648         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1649         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1650         verify_media_type($opt, $vtype, $drive->{media});
1651         $drive->{file} = $volid;
1652     }
1653
1654     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1655 }
1656
1657 sub parse_hotplug_features {
1658     my ($data) = @_;
1659
1660     my $res = {};
1661
1662     return $res if $data eq '0';
1663
1664     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1665
1666     foreach my $feature (PVE::Tools::split_list($data)) {
1667         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1668             $res->{$1} = 1;
1669         } else {
1670             die "invalid hotplug feature '$feature'\n";
1671         }
1672     }
1673     return $res;
1674 }
1675
1676 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1677 sub pve_verify_hotplug_features {
1678     my ($value, $noerr) = @_;
1679
1680     return $value if parse_hotplug_features($value);
1681
1682     return undef if $noerr;
1683
1684     die "unable to parse hotplug option\n";
1685 }
1686
1687 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1688 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1689 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1690 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1691 #        [,iothread=on][,serial=serial][,model=model]
1692
1693 sub parse_drive {
1694     my ($key, $data) = @_;
1695
1696     my ($interface, $index);
1697
1698     if ($key =~ m/^([^\d]+)(\d+)$/) {
1699         $interface = $1;
1700         $index = $2;
1701     } else {
1702         return undef;
1703     }
1704
1705     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1706                                      : $confdesc->{$key}->{format};
1707     if (!$desc) {
1708         warn "invalid drive key: $key\n";
1709         return undef;
1710     }
1711     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1712     return undef if !$res;
1713     $res->{interface} = $interface;
1714     $res->{index} = $index;
1715
1716     my $error = 0;
1717     foreach my $opt (qw(bps bps_rd bps_wr)) {
1718         if (my $bps = defined(delete $res->{$opt})) {
1719             if (defined($res->{"m$opt"})) {
1720                 warn "both $opt and m$opt specified\n";
1721                 ++$error;
1722                 next;
1723             }
1724             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1725         }
1726     }
1727
1728     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1729     for my $requirement (
1730         [mbps_max => 'mbps'],
1731         [mbps_rd_max => 'mbps_rd'],
1732         [mbps_wr_max => 'mbps_wr'],
1733         [miops_max => 'miops'],
1734         [miops_rd_max => 'miops_rd'],
1735         [miops_wr_max => 'miops_wr'],
1736         [bps_max_length => 'mbps_max'],
1737         [bps_rd_max_length => 'mbps_rd_max'],
1738         [bps_wr_max_length => 'mbps_wr_max'],
1739         [iops_max_length => 'iops_max'],
1740         [iops_rd_max_length => 'iops_rd_max'],
1741         [iops_wr_max_length => 'iops_wr_max']) {
1742         my ($option, $requires) = @$requirement;
1743         if ($res->{$option} && !$res->{$requires}) {
1744             warn "$option requires $requires\n";
1745             ++$error;
1746         }
1747     }
1748
1749     return undef if $error;
1750
1751     return undef if $res->{mbps_rd} && $res->{mbps};
1752     return undef if $res->{mbps_wr} && $res->{mbps};
1753     return undef if $res->{iops_rd} && $res->{iops};
1754     return undef if $res->{iops_wr} && $res->{iops};
1755
1756     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1757         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1758         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1759         return undef if $res->{interface} eq 'virtio';
1760     }
1761
1762     if (my $size = $res->{size}) {
1763         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1764     }
1765
1766     return $res;
1767 }
1768
1769 sub print_drive {
1770     my ($vmid, $drive) = @_;
1771     my $data = { %$drive };
1772     delete $data->{$_} for qw(index interface);
1773     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1774 }
1775
1776 sub scsi_inquiry {
1777     my($fh, $noerr) = @_;
1778
1779     my $SG_IO = 0x2285;
1780     my $SG_GET_VERSION_NUM = 0x2282;
1781
1782     my $versionbuf = "\x00" x 8;
1783     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1784     if (!$ret) {
1785         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1786         return undef;
1787     }
1788     my $version = unpack("I", $versionbuf);
1789     if ($version < 30000) {
1790         die "scsi generic interface too old\n"  if !$noerr;
1791         return undef;
1792     }
1793
1794     my $buf = "\x00" x 36;
1795     my $sensebuf = "\x00" x 8;
1796     my $cmd = pack("C x3 C x1", 0x12, 36);
1797
1798     # see /usr/include/scsi/sg.h
1799     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1800
1801     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1802                       length($sensebuf), 0, length($buf), $buf,
1803                       $cmd, $sensebuf, 6000);
1804
1805     $ret = ioctl($fh, $SG_IO, $packet);
1806     if (!$ret) {
1807         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1808         return undef;
1809     }
1810
1811     my @res = unpack($sg_io_hdr_t, $packet);
1812     if ($res[17] || $res[18]) {
1813         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1814         return undef;
1815     }
1816
1817     my $res = {};
1818     (my $byte0, my $byte1, $res->{vendor},
1819      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1820
1821     $res->{removable} = $byte1 & 128 ? 1 : 0;
1822     $res->{type} = $byte0 & 31;
1823
1824     return $res;
1825 }
1826
1827 sub path_is_scsi {
1828     my ($path) = @_;
1829
1830     my $fh = IO::File->new("+<$path") || return undef;
1831     my $res = scsi_inquiry($fh, 1);
1832     close($fh);
1833
1834     return $res;
1835 }
1836
1837 sub machine_type_is_q35 {
1838     my ($conf) = @_;
1839
1840     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1841 }
1842
1843 sub print_tabletdevice_full {
1844     my ($conf, $arch) = @_;
1845
1846     my $q35 = machine_type_is_q35($conf);
1847
1848     # we use uhci for old VMs because tablet driver was buggy in older qemu
1849     my $usbbus;
1850     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1851         $usbbus = 'ehci';
1852     } else {
1853         $usbbus = 'uhci';
1854     }
1855
1856     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1857 }
1858
1859 sub print_keyboarddevice_full {
1860     my ($conf, $arch, $machine) = @_;
1861
1862     return undef if $arch ne 'aarch64';
1863
1864     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1865 }
1866
1867 sub print_drivedevice_full {
1868     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1869
1870     my $device = '';
1871     my $maxdev = 0;
1872
1873     if ($drive->{interface} eq 'virtio') {
1874         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1875         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1876         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1877     } elsif ($drive->{interface} eq 'scsi') {
1878
1879         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1880         my $unit = $drive->{index} % $maxdev;
1881         my $devicetype = 'hd';
1882         my $path = '';
1883         if (drive_is_cdrom($drive)) {
1884             $devicetype = 'cd';
1885         } else {
1886             if ($drive->{file} =~ m|^/|) {
1887                 $path = $drive->{file};
1888                 if (my $info = path_is_scsi($path)) {
1889                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1890                         $devicetype = 'block';
1891                     } elsif ($info->{type} == 1) { # tape
1892                         $devicetype = 'generic';
1893                     }
1894                 }
1895             } else {
1896                  $path = PVE::Storage::path($storecfg, $drive->{file});
1897             }
1898
1899             if($path =~ m/^iscsi\:\/\//){
1900                 $devicetype = 'generic';
1901             }
1902         }
1903
1904         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1905             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1906         } else {
1907             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1908         }
1909
1910         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1911             $device .= ",rotation_rate=1";
1912         }
1913         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1914
1915     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1916         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1917         my $controller = int($drive->{index} / $maxdev);
1918         my $unit = $drive->{index} % $maxdev;
1919         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1920
1921         $device = "ide-$devicetype";
1922         if ($drive->{interface} eq 'ide') {
1923             $device .= ",bus=ide.$controller,unit=$unit";
1924         } else {
1925             $device .= ",bus=ahci$controller.$unit";
1926         }
1927         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1928
1929         if ($devicetype eq 'hd') {
1930             if (my $model = $drive->{model}) {
1931                 $model = URI::Escape::uri_unescape($model);
1932                 $device .= ",model=$model";
1933             }
1934             if ($drive->{ssd}) {
1935                 $device .= ",rotation_rate=1";
1936             }
1937         }
1938         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1939     } elsif ($drive->{interface} eq 'usb') {
1940         die "implement me";
1941         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1942     } else {
1943         die "unsupported interface type";
1944     }
1945
1946     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1947
1948     if (my $serial = $drive->{serial}) {
1949         $serial = URI::Escape::uri_unescape($serial);
1950         $device .= ",serial=$serial";
1951     }
1952
1953
1954     return $device;
1955 }
1956
1957 sub get_initiator_name {
1958     my $initiator;
1959
1960     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1961     while (defined(my $line = <$fh>)) {
1962         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1963         $initiator = $1;
1964         last;
1965     }
1966     $fh->close();
1967
1968     return $initiator;
1969 }
1970
1971 sub print_drive_full {
1972     my ($storecfg, $vmid, $drive) = @_;
1973
1974     my $path;
1975     my $volid = $drive->{file};
1976     my $format;
1977
1978     if (drive_is_cdrom($drive)) {
1979         $path = get_iso_path($storecfg, $vmid, $volid);
1980     } else {
1981         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1982         if ($storeid) {
1983             $path = PVE::Storage::path($storecfg, $volid);
1984             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1985             $format = qemu_img_format($scfg, $volname);
1986         } else {
1987             $path = $volid;
1988             $format = "raw";
1989         }
1990    }
1991
1992     my $opts = '';
1993     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1994     foreach my $o (@qemu_drive_options) {
1995         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1996     }
1997
1998     # snapshot only accepts on|off
1999     if (defined($drive->{snapshot})) {
2000         my $v = $drive->{snapshot} ? 'on' : 'off';
2001         $opts .= ",snapshot=$v";
2002     }
2003
2004     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2005         my ($dir, $qmpname) = @$type;
2006         if (my $v = $drive->{"mbps$dir"}) {
2007             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2008         }
2009         if (my $v = $drive->{"mbps${dir}_max"}) {
2010             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2011         }
2012         if (my $v = $drive->{"bps${dir}_max_length"}) {
2013             $opts .= ",throttling.bps$qmpname-max-length=$v";
2014         }
2015         if (my $v = $drive->{"iops${dir}"}) {
2016             $opts .= ",throttling.iops$qmpname=$v";
2017         }
2018         if (my $v = $drive->{"iops${dir}_max"}) {
2019             $opts .= ",throttling.iops$qmpname-max=$v";
2020         }
2021         if (my $v = $drive->{"iops${dir}_max_length"}) {
2022             $opts .= ",throttling.iops$qmpname-max-length=$v";
2023         }
2024     }
2025
2026     $opts .= ",format=$format" if $format && !$drive->{format};
2027
2028     my $cache_direct = 0;
2029
2030     if (my $cache = $drive->{cache}) {
2031         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2032     } elsif (!drive_is_cdrom($drive)) {
2033         $opts .= ",cache=none";
2034         $cache_direct = 1;
2035     }
2036
2037     # aio native works only with O_DIRECT
2038     if (!$drive->{aio}) {
2039         if($cache_direct) {
2040             $opts .= ",aio=native";
2041         } else {
2042             $opts .= ",aio=threads";
2043         }
2044     }
2045
2046     if (!drive_is_cdrom($drive)) {
2047         my $detectzeroes;
2048         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2049             $detectzeroes = 'off';
2050         } elsif ($drive->{discard}) {
2051             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2052         } else {
2053             # This used to be our default with discard not being specified:
2054             $detectzeroes = 'on';
2055         }
2056         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2057     }
2058
2059     my $pathinfo = $path ? "file=$path," : '';
2060
2061     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2062 }
2063
2064 sub print_netdevice_full {
2065     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2066
2067     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2068
2069     my $device = $net->{model};
2070     if ($net->{model} eq 'virtio') {
2071          $device = 'virtio-net-pci';
2072      };
2073
2074     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2075     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2076     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2077         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2078         my $vectors = $net->{queues} * 2 + 2;
2079         $tmpstr .= ",vectors=$vectors,mq=on";
2080     }
2081     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2082
2083     if ($use_old_bios_files) {
2084         my $romfile;
2085         if ($device eq 'virtio-net-pci') {
2086             $romfile = 'pxe-virtio.rom';
2087         } elsif ($device eq 'e1000') {
2088             $romfile = 'pxe-e1000.rom';
2089         } elsif ($device eq 'ne2k') {
2090             $romfile = 'pxe-ne2k_pci.rom';
2091         } elsif ($device eq 'pcnet') {
2092             $romfile = 'pxe-pcnet.rom';
2093         } elsif ($device eq 'rtl8139') {
2094             $romfile = 'pxe-rtl8139.rom';
2095         }
2096         $tmpstr .= ",romfile=$romfile" if $romfile;
2097     }
2098
2099     return $tmpstr;
2100 }
2101
2102 sub print_netdev_full {
2103     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2104
2105     my $i = '';
2106     if ($netid =~ m/^net(\d+)$/) {
2107         $i = int($1);
2108     }
2109
2110     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2111
2112     my $ifname = "tap${vmid}i$i";
2113
2114     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2115     die "interface name '$ifname' is too long (max 15 character)\n"
2116         if length($ifname) >= 16;
2117
2118     my $vhostparam = '';
2119     if (is_native($arch)) {
2120         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2121     }
2122
2123     my $vmname = $conf->{name} || "vm$vmid";
2124
2125     my $netdev = "";
2126     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2127
2128     if ($net->{bridge}) {
2129         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2130     } else {
2131         $netdev = "type=user,id=$netid,hostname=$vmname";
2132     }
2133
2134     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2135
2136     return $netdev;
2137 }
2138
2139
2140 sub print_cpu_device {
2141     my ($conf, $id) = @_;
2142
2143     my $kvm = $conf->{kvm} // 1;
2144     my $cpu = $kvm ? "kvm64" : "qemu64";
2145     if (my $cputype = $conf->{cpu}) {
2146         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2147             or die "Cannot parse cpu description: $cputype\n";
2148         $cpu = $cpuconf->{cputype};
2149     }
2150
2151     my $cores = $conf->{cores} || 1;
2152
2153     my $current_core = ($id - 1) % $cores;
2154     my $current_socket = int(($id - 1 - $current_core)/$cores);
2155
2156     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2157 }
2158
2159 my $vga_map = {
2160     'cirrus' => 'cirrus-vga',
2161     'std' => 'VGA',
2162     'vmware' => 'vmware-svga',
2163     'virtio' => 'virtio-vga',
2164 };
2165
2166 sub print_vga_device {
2167     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2168
2169     my $type = $vga_map->{$vga->{type}};
2170     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2171         $type = 'virtio-gpu';
2172     }
2173     my $vgamem_mb = $vga->{memory};
2174     if ($qxlnum) {
2175         $type = $id ? 'qxl' : 'qxl-vga';
2176     }
2177     die "no devicetype for $vga->{type}\n" if !$type;
2178
2179     my $memory = "";
2180     if ($vgamem_mb) {
2181         if ($vga->{type} eq 'virtio') {
2182             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2183             $memory = ",max_hostmem=$bytes";
2184         } elsif ($qxlnum) {
2185             # from https://www.spice-space.org/multiple-monitors.html
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187             my $ram = $vgamem_mb * 4;
2188             my $vram = $vgamem_mb * 2;
2189             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2190         } else {
2191             $memory = ",vgamem_mb=$vga->{memory}";
2192         }
2193     } elsif ($qxlnum && $id) {
2194         $memory = ",ram_size=67108864,vram_size=33554432";
2195     }
2196
2197     my $q35 = machine_type_is_q35($conf);
2198     my $vgaid = "vga" . ($id // '');
2199     my $pciaddr;
2200
2201     if ($q35 && $vgaid eq 'vga') {
2202         # the first display uses pcie.0 bus on q35 machines
2203         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2204     } else {
2205         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2206     }
2207
2208     return "$type,id=${vgaid}${memory}${pciaddr}";
2209 }
2210
2211 sub drive_is_cloudinit {
2212     my ($drive) = @_;
2213     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2214 }
2215
2216 sub drive_is_cdrom {
2217     my ($drive, $exclude_cloudinit) = @_;
2218
2219     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2220
2221     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2222
2223 }
2224
2225 sub parse_number_sets {
2226     my ($set) = @_;
2227     my $res = [];
2228     foreach my $part (split(/;/, $set)) {
2229         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2230             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2231             push @$res, [ $1, $2 ];
2232         } else {
2233             die "invalid range: $part\n";
2234         }
2235     }
2236     return $res;
2237 }
2238
2239 sub parse_numa {
2240     my ($data) = @_;
2241
2242     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2243     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2244     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2245     return $res;
2246 }
2247
2248 sub parse_hostpci {
2249     my ($value) = @_;
2250
2251     return undef if !$value;
2252
2253     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2254
2255     my @idlist = split(/;/, $res->{host});
2256     delete $res->{host};
2257     foreach my $id (@idlist) {
2258         if ($id =~ m/\./) { # full id 00:00.1
2259             push @{$res->{pciid}}, {
2260                 id => $id,
2261             };
2262         } else { # partial id 00:00
2263             $res->{pciid} = PVE::SysFSTools::lspci($id);
2264         }
2265     }
2266     return $res;
2267 }
2268
2269 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2270 sub parse_net {
2271     my ($data) = @_;
2272
2273     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2274     if ($@) {
2275         warn $@;
2276         return undef;
2277     }
2278     if (!defined($res->{macaddr})) {
2279         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2280         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2281     }
2282     return $res;
2283 }
2284
2285 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2286 sub parse_ipconfig {
2287     my ($data) = @_;
2288
2289     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2290     if ($@) {
2291         warn $@;
2292         return undef;
2293     }
2294
2295     if ($res->{gw} && !$res->{ip}) {
2296         warn 'gateway specified without specifying an IP address';
2297         return undef;
2298     }
2299     if ($res->{gw6} && !$res->{ip6}) {
2300         warn 'IPv6 gateway specified without specifying an IPv6 address';
2301         return undef;
2302     }
2303     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2304         warn 'gateway specified together with DHCP';
2305         return undef;
2306     }
2307     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2308         # gw6 + auto/dhcp
2309         warn "IPv6 gateway specified together with $res->{ip6} address";
2310         return undef;
2311     }
2312
2313     if (!$res->{ip} && !$res->{ip6}) {
2314         return { ip => 'dhcp', ip6 => 'dhcp' };
2315     }
2316
2317     return $res;
2318 }
2319
2320 sub print_net {
2321     my $net = shift;
2322
2323     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2324 }
2325
2326 sub add_random_macs {
2327     my ($settings) = @_;
2328
2329     foreach my $opt (keys %$settings) {
2330         next if $opt !~ m/^net(\d+)$/;
2331         my $net = parse_net($settings->{$opt});
2332         next if !$net;
2333         $settings->{$opt} = print_net($net);
2334     }
2335 }
2336
2337 sub vm_is_volid_owner {
2338     my ($storecfg, $vmid, $volid) = @_;
2339
2340     if ($volid !~  m|^/|) {
2341         my ($path, $owner);
2342         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2343         if ($owner && ($owner == $vmid)) {
2344             return 1;
2345         }
2346     }
2347
2348     return undef;
2349 }
2350
2351 sub split_flagged_list {
2352     my $text = shift || '';
2353     $text =~ s/[,;]/ /g;
2354     $text =~ s/^\s+//;
2355     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2356 }
2357
2358 sub join_flagged_list {
2359     my ($how, $lst) = @_;
2360     join $how, map { $lst->{$_} . $_ } keys %$lst;
2361 }
2362
2363 sub vmconfig_delete_pending_option {
2364     my ($conf, $key, $force) = @_;
2365
2366     delete $conf->{pending}->{$key};
2367     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2368     $pending_delete_hash->{$key} = $force ? '!' : '';
2369     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2370 }
2371
2372 sub vmconfig_undelete_pending_option {
2373     my ($conf, $key) = @_;
2374
2375     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2376     delete $pending_delete_hash->{$key};
2377
2378     if (%$pending_delete_hash) {
2379         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2380     } else {
2381         delete $conf->{pending}->{delete};
2382     }
2383 }
2384
2385 sub vmconfig_register_unused_drive {
2386     my ($storecfg, $vmid, $conf, $drive) = @_;
2387
2388     if (drive_is_cloudinit($drive)) {
2389         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2390         warn $@ if $@;
2391     } elsif (!drive_is_cdrom($drive)) {
2392         my $volid = $drive->{file};
2393         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2394             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2395         }
2396     }
2397 }
2398
2399 sub vmconfig_cleanup_pending {
2400     my ($conf) = @_;
2401
2402     # remove pending changes when nothing changed
2403     my $changes;
2404     foreach my $opt (keys %{$conf->{pending}}) {
2405         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2406             $changes = 1;
2407             delete $conf->{pending}->{$opt};
2408         }
2409     }
2410
2411     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2412     my $pending_delete_hash = {};
2413     while (my ($opt, $force) = each %$current_delete_hash) {
2414         if (defined($conf->{$opt})) {
2415             $pending_delete_hash->{$opt} = $force;
2416         } else {
2417             $changes = 1;
2418         }
2419     }
2420
2421     if (%$pending_delete_hash) {
2422         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2423     } else {
2424         delete $conf->{pending}->{delete};
2425     }
2426
2427     return $changes;
2428 }
2429
2430 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2431 my $smbios1_fmt = {
2432     uuid => {
2433         type => 'string',
2434         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2435         format_description => 'UUID',
2436         description => "Set SMBIOS1 UUID.",
2437         optional => 1,
2438     },
2439     version => {
2440         type => 'string',
2441         pattern => '[A-Za-z0-9+\/]+={0,2}',
2442         format_description => 'Base64 encoded string',
2443         description => "Set SMBIOS1 version.",
2444         optional => 1,
2445     },
2446     serial => {
2447         type => 'string',
2448         pattern => '[A-Za-z0-9+\/]+={0,2}',
2449         format_description => 'Base64 encoded string',
2450         description => "Set SMBIOS1 serial number.",
2451         optional => 1,
2452     },
2453     manufacturer => {
2454         type => 'string',
2455         pattern => '[A-Za-z0-9+\/]+={0,2}',
2456         format_description => 'Base64 encoded string',
2457         description => "Set SMBIOS1 manufacturer.",
2458         optional => 1,
2459     },
2460     product => {
2461         type => 'string',
2462         pattern => '[A-Za-z0-9+\/]+={0,2}',
2463         format_description => 'Base64 encoded string',
2464         description => "Set SMBIOS1 product ID.",
2465         optional => 1,
2466     },
2467     sku => {
2468         type => 'string',
2469         pattern => '[A-Za-z0-9+\/]+={0,2}',
2470         format_description => 'Base64 encoded string',
2471         description => "Set SMBIOS1 SKU string.",
2472         optional => 1,
2473     },
2474     family => {
2475         type => 'string',
2476         pattern => '[A-Za-z0-9+\/]+={0,2}',
2477         format_description => 'Base64 encoded string',
2478         description => "Set SMBIOS1 family string.",
2479         optional => 1,
2480     },
2481     base64 => {
2482         type => 'boolean',
2483         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2484         optional => 1,
2485     },
2486 };
2487
2488 sub parse_smbios1 {
2489     my ($data) = @_;
2490
2491     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2492     warn $@ if $@;
2493     return $res;
2494 }
2495
2496 sub print_smbios1 {
2497     my ($smbios1) = @_;
2498     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2499 }
2500
2501 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2502
2503 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2504 sub verify_bootdisk {
2505     my ($value, $noerr) = @_;
2506
2507     return $value if is_valid_drivename($value);
2508
2509     return undef if $noerr;
2510
2511     die "invalid boot disk '$value'\n";
2512 }
2513
2514 sub parse_watchdog {
2515     my ($value) = @_;
2516
2517     return undef if !$value;
2518
2519     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2520     warn $@ if $@;
2521     return $res;
2522 }
2523
2524 sub parse_guest_agent {
2525     my ($value) = @_;
2526
2527     return {} if !defined($value->{agent});
2528
2529     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2530     warn $@ if $@;
2531
2532     # if the agent is disabled ignore the other potentially set properties
2533     return {} if !$res->{enabled};
2534     return $res;
2535 }
2536
2537 sub parse_vga {
2538     my ($value) = @_;
2539
2540     return {} if !$value;
2541     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2542     warn $@ if $@;
2543     return $res;
2544 }
2545
2546 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2547 sub verify_usb_device {
2548     my ($value, $noerr) = @_;
2549
2550     return $value if parse_usb_device($value);
2551
2552     return undef if $noerr;
2553
2554     die "unable to parse usb device\n";
2555 }
2556
2557 # add JSON properties for create and set function
2558 sub json_config_properties {
2559     my $prop = shift;
2560
2561     foreach my $opt (keys %$confdesc) {
2562         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2563         $prop->{$opt} = $confdesc->{$opt};
2564     }
2565
2566     return $prop;
2567 }
2568
2569 # return copy of $confdesc_cloudinit to generate documentation
2570 sub cloudinit_config_properties {
2571
2572     return dclone($confdesc_cloudinit);
2573 }
2574
2575 sub check_type {
2576     my ($key, $value) = @_;
2577
2578     die "unknown setting '$key'\n" if !$confdesc->{$key};
2579
2580     my $type = $confdesc->{$key}->{type};
2581
2582     if (!defined($value)) {
2583         die "got undefined value\n";
2584     }
2585
2586     if ($value =~ m/[\n\r]/) {
2587         die "property contains a line feed\n";
2588     }
2589
2590     if ($type eq 'boolean') {
2591         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2592         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2593         die "type check ('boolean') failed - got '$value'\n";
2594     } elsif ($type eq 'integer') {
2595         return int($1) if $value =~ m/^(\d+)$/;
2596         die "type check ('integer') failed - got '$value'\n";
2597     } elsif ($type eq 'number') {
2598         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2599         die "type check ('number') failed - got '$value'\n";
2600     } elsif ($type eq 'string') {
2601         if (my $fmt = $confdesc->{$key}->{format}) {
2602             PVE::JSONSchema::check_format($fmt, $value);
2603             return $value;
2604         }
2605         $value =~ s/^\"(.*)\"$/$1/;
2606         return $value;
2607     } else {
2608         die "internal error"
2609     }
2610 }
2611
2612 sub touch_config {
2613     my ($vmid) = @_;
2614
2615     my $conf = PVE::QemuConfig->config_file($vmid);
2616     utime undef, undef, $conf;
2617 }
2618
2619 sub destroy_vm {
2620     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2621
2622     my $conffile = PVE::QemuConfig->config_file($vmid);
2623
2624     my $conf = PVE::QemuConfig->load_config($vmid);
2625
2626     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2627
2628     if ($conf->{template}) {
2629         # check if any base image is still used by a linked clone
2630         foreach_drive($conf, sub {
2631                 my ($ds, $drive) = @_;
2632
2633                 return if drive_is_cdrom($drive);
2634
2635                 my $volid = $drive->{file};
2636
2637                 return if !$volid || $volid =~ m|^/|;
2638
2639                 die "base volume '$volid' is still in use by linked cloned\n"
2640                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2641
2642         });
2643     }
2644
2645     # only remove disks owned by this VM
2646     foreach_drive($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         return if drive_is_cdrom($drive, 1);
2650
2651         my $volid = $drive->{file};
2652
2653         return if !$volid || $volid =~ m|^/|;
2654
2655         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2656         return if !$path || !$owner || ($owner != $vmid);
2657
2658         eval {
2659             PVE::Storage::vdisk_free($storecfg, $volid);
2660         };
2661         warn "Could not remove disk '$volid', check manually: $@" if $@;
2662
2663     });
2664
2665     if ($keep_empty_config) {
2666         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2667     } else {
2668         unlink $conffile;
2669     }
2670
2671     # also remove unused disk
2672     eval {
2673         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2674
2675         eval {
2676             PVE::Storage::foreach_volid($dl, sub {
2677                 my ($volid, $sid, $volname, $d) = @_;
2678                 PVE::Storage::vdisk_free($storecfg, $volid);
2679             });
2680         };
2681         warn $@ if $@;
2682
2683     };
2684     warn $@ if $@;
2685 }
2686
2687 sub parse_vm_config {
2688     my ($filename, $raw) = @_;
2689
2690     return undef if !defined($raw);
2691
2692     my $res = {
2693         digest => Digest::SHA::sha1_hex($raw),
2694         snapshots => {},
2695         pending => {},
2696     };
2697
2698     $filename =~ m|/qemu-server/(\d+)\.conf$|
2699         || die "got strange filename '$filename'";
2700
2701     my $vmid = $1;
2702
2703     my $conf = $res;
2704     my $descr;
2705     my $section = '';
2706
2707     my @lines = split(/\n/, $raw);
2708     foreach my $line (@lines) {
2709         next if $line =~ m/^\s*$/;
2710
2711         if ($line =~ m/^\[PENDING\]\s*$/i) {
2712             $section = 'pending';
2713             if (defined($descr)) {
2714                 $descr =~ s/\s+$//;
2715                 $conf->{description} = $descr;
2716             }
2717             $descr = undef;
2718             $conf = $res->{$section} = {};
2719             next;
2720
2721         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2722             $section = $1;
2723             if (defined($descr)) {
2724                 $descr =~ s/\s+$//;
2725                 $conf->{description} = $descr;
2726             }
2727             $descr = undef;
2728             $conf = $res->{snapshots}->{$section} = {};
2729             next;
2730         }
2731
2732         if ($line =~ m/^\#(.*)\s*$/) {
2733             $descr = '' if !defined($descr);
2734             $descr .= PVE::Tools::decode_text($1) . "\n";
2735             next;
2736         }
2737
2738         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2739             $descr = '' if !defined($descr);
2740             $descr .= PVE::Tools::decode_text($2);
2741         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2742             $conf->{snapstate} = $1;
2743         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2744             my $key = $1;
2745             my $value = $2;
2746             $conf->{$key} = $value;
2747         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2748             my $value = $1;
2749             if ($section eq 'pending') {
2750                 $conf->{delete} = $value; # we parse this later
2751             } else {
2752                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2753             }
2754         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2755             my $key = $1;
2756             my $value = $2;
2757             eval { $value = check_type($key, $value); };
2758             if ($@) {
2759                 warn "vm $vmid - unable to parse value of '$key' - $@";
2760             } else {
2761                 $key = 'ide2' if $key eq 'cdrom';
2762                 my $fmt = $confdesc->{$key}->{format};
2763                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2764                     my $v = parse_drive($key, $value);
2765                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2766                         $v->{file} = $volid;
2767                         $value = print_drive($vmid, $v);
2768                     } else {
2769                         warn "vm $vmid - unable to parse value of '$key'\n";
2770                         next;
2771                     }
2772                 }
2773
2774                 $conf->{$key} = $value;
2775             }
2776         }
2777     }
2778
2779     if (defined($descr)) {
2780         $descr =~ s/\s+$//;
2781         $conf->{description} = $descr;
2782     }
2783     delete $res->{snapstate}; # just to be sure
2784
2785     return $res;
2786 }
2787
2788 sub write_vm_config {
2789     my ($filename, $conf) = @_;
2790
2791     delete $conf->{snapstate}; # just to be sure
2792
2793     if ($conf->{cdrom}) {
2794         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2795         $conf->{ide2} = $conf->{cdrom};
2796         delete $conf->{cdrom};
2797     }
2798
2799     # we do not use 'smp' any longer
2800     if ($conf->{sockets}) {
2801         delete $conf->{smp};
2802     } elsif ($conf->{smp}) {
2803         $conf->{sockets} = $conf->{smp};
2804         delete $conf->{cores};
2805         delete $conf->{smp};
2806     }
2807
2808     my $used_volids = {};
2809
2810     my $cleanup_config = sub {
2811         my ($cref, $pending, $snapname) = @_;
2812
2813         foreach my $key (keys %$cref) {
2814             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2815                 $key eq 'snapstate' || $key eq 'pending';
2816             my $value = $cref->{$key};
2817             if ($key eq 'delete') {
2818                 die "propertry 'delete' is only allowed in [PENDING]\n"
2819                     if !$pending;
2820                 # fixme: check syntax?
2821                 next;
2822             }
2823             eval { $value = check_type($key, $value); };
2824             die "unable to parse value of '$key' - $@" if $@;
2825
2826             $cref->{$key} = $value;
2827
2828             if (!$snapname && is_valid_drivename($key)) {
2829                 my $drive = parse_drive($key, $value);
2830                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2831             }
2832         }
2833     };
2834
2835     &$cleanup_config($conf);
2836
2837     &$cleanup_config($conf->{pending}, 1);
2838
2839     foreach my $snapname (keys %{$conf->{snapshots}}) {
2840         die "internal error" if $snapname eq 'pending';
2841         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2842     }
2843
2844     # remove 'unusedX' settings if we re-add a volume
2845     foreach my $key (keys %$conf) {
2846         my $value = $conf->{$key};
2847         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2848             delete $conf->{$key};
2849         }
2850     }
2851
2852     my $generate_raw_config = sub {
2853         my ($conf, $pending) = @_;
2854
2855         my $raw = '';
2856
2857         # add description as comment to top of file
2858         if (defined(my $descr = $conf->{description})) {
2859             if ($descr) {
2860                 foreach my $cl (split(/\n/, $descr)) {
2861                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2862                 }
2863             } else {
2864                 $raw .= "#\n" if $pending;
2865             }
2866         }
2867
2868         foreach my $key (sort keys %$conf) {
2869             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2870             $raw .= "$key: $conf->{$key}\n";
2871         }
2872         return $raw;
2873     };
2874
2875     my $raw = &$generate_raw_config($conf);
2876
2877     if (scalar(keys %{$conf->{pending}})){
2878         $raw .= "\n[PENDING]\n";
2879         $raw .= &$generate_raw_config($conf->{pending}, 1);
2880     }
2881
2882     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2883         $raw .= "\n[$snapname]\n";
2884         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2885     }
2886
2887     return $raw;
2888 }
2889
2890 sub load_defaults {
2891
2892     my $res = {};
2893
2894     # we use static defaults from our JSON schema configuration
2895     foreach my $key (keys %$confdesc) {
2896         if (defined(my $default = $confdesc->{$key}->{default})) {
2897             $res->{$key} = $default;
2898         }
2899     }
2900
2901     return $res;
2902 }
2903
2904 sub config_list {
2905     my $vmlist = PVE::Cluster::get_vmlist();
2906     my $res = {};
2907     return $res if !$vmlist || !$vmlist->{ids};
2908     my $ids = $vmlist->{ids};
2909
2910     foreach my $vmid (keys %$ids) {
2911         my $d = $ids->{$vmid};
2912         next if !$d->{node} || $d->{node} ne $nodename;
2913         next if !$d->{type} || $d->{type} ne 'qemu';
2914         $res->{$vmid}->{exists} = 1;
2915     }
2916     return $res;
2917 }
2918
2919 # test if VM uses local resources (to prevent migration)
2920 sub check_local_resources {
2921     my ($conf, $noerr) = @_;
2922
2923     my @loc_res = ();
2924
2925     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2926     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2927
2928     push @loc_res, "ivshmem" if $conf->{ivshmem};
2929
2930     foreach my $k (keys %$conf) {
2931         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2932         # sockets are safe: they will recreated be on the target side post-migrate
2933         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2934         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2935     }
2936
2937     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2938
2939     return \@loc_res;
2940 }
2941
2942 # check if used storages are available on all nodes (use by migrate)
2943 sub check_storage_availability {
2944     my ($storecfg, $conf, $node) = @_;
2945
2946     foreach_drive($conf, sub {
2947         my ($ds, $drive) = @_;
2948
2949         my $volid = $drive->{file};
2950         return if !$volid;
2951
2952         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2953         return if !$sid;
2954
2955         # check if storage is available on both nodes
2956         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2957         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2958    });
2959 }
2960
2961 # list nodes where all VM images are available (used by has_feature API)
2962 sub shared_nodes {
2963     my ($conf, $storecfg) = @_;
2964
2965     my $nodelist = PVE::Cluster::get_nodelist();
2966     my $nodehash = { map { $_ => 1 } @$nodelist };
2967     my $nodename = PVE::INotify::nodename();
2968
2969     foreach_drive($conf, sub {
2970         my ($ds, $drive) = @_;
2971
2972         my $volid = $drive->{file};
2973         return if !$volid;
2974
2975         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2976         if ($storeid) {
2977             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2978             if ($scfg->{disable}) {
2979                 $nodehash = {};
2980             } elsif (my $avail = $scfg->{nodes}) {
2981                 foreach my $node (keys %$nodehash) {
2982                     delete $nodehash->{$node} if !$avail->{$node};
2983                 }
2984             } elsif (!$scfg->{shared}) {
2985                 foreach my $node (keys %$nodehash) {
2986                     delete $nodehash->{$node} if $node ne $nodename
2987                 }
2988             }
2989         }
2990     });
2991
2992     return $nodehash
2993 }
2994
2995 sub check_local_storage_availability {
2996     my ($conf, $storecfg) = @_;
2997
2998     my $nodelist = PVE::Cluster::get_nodelist();
2999     my $nodehash = { map { $_ => {} } @$nodelist };
3000
3001     foreach_drive($conf, sub {
3002         my ($ds, $drive) = @_;
3003
3004         my $volid = $drive->{file};
3005         return if !$volid;
3006
3007         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3008         if ($storeid) {
3009             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3010
3011             if ($scfg->{disable}) {
3012                 foreach my $node (keys %$nodehash) {
3013                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3014                 }
3015             } elsif (my $avail = $scfg->{nodes}) {
3016                 foreach my $node (keys %$nodehash) {
3017                     if (!$avail->{$node}) {
3018                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3019                     }
3020                 }
3021             }
3022         }
3023     });
3024
3025     foreach my $node (values %$nodehash) {
3026         if (my $unavail = $node->{unavailable_storages}) {
3027             $node->{unavailable_storages} = [ sort keys %$unavail ];
3028         }
3029     }
3030
3031     return $nodehash
3032 }
3033
3034 sub check_cmdline {
3035     my ($pidfile, $pid) = @_;
3036
3037     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3038     if (defined($fh)) {
3039         my $line = <$fh>;
3040         $fh->close;
3041         return undef if !$line;
3042         my @param = split(/\0/, $line);
3043
3044         my $cmd = $param[0];
3045         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3046
3047         for (my $i = 0; $i < scalar (@param); $i++) {
3048             my $p = $param[$i];
3049             next if !$p;
3050             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3051                 my $p = $param[$i+1];
3052                 return 1 if $p && ($p eq $pidfile);
3053                 return undef;
3054             }
3055         }
3056     }
3057     return undef;
3058 }
3059
3060 sub check_running {
3061     my ($vmid, $nocheck, $node) = @_;
3062
3063     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3064
3065     die "unable to find configuration file for VM $vmid - no such machine\n"
3066         if !$nocheck && ! -f $filename;
3067
3068     my $pidfile = pidfile_name($vmid);
3069
3070     if (my $fd = IO::File->new("<$pidfile")) {
3071         my $st = stat($fd);
3072         my $line = <$fd>;
3073         close($fd);
3074
3075         my $mtime = $st->mtime;
3076         if ($mtime > time()) {
3077             warn "file '$filename' modified in future\n";
3078         }
3079
3080         if ($line =~ m/^(\d+)$/) {
3081             my $pid = $1;
3082             if (check_cmdline($pidfile, $pid)) {
3083                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3084                     return $pid;
3085                 }
3086             }
3087         }
3088     }
3089
3090     return undef;
3091 }
3092
3093 sub vzlist {
3094
3095     my $vzlist = config_list();
3096
3097     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3098
3099     while (defined(my $de = $fd->read)) {
3100         next if $de !~ m/^(\d+)\.pid$/;
3101         my $vmid = $1;
3102         next if !defined($vzlist->{$vmid});
3103         if (my $pid = check_running($vmid)) {
3104             $vzlist->{$vmid}->{pid} = $pid;
3105         }
3106     }
3107
3108     return $vzlist;
3109 }
3110
3111 sub disksize {
3112     my ($storecfg, $conf) = @_;
3113
3114     my $bootdisk = $conf->{bootdisk};
3115     return undef if !$bootdisk;
3116     return undef if !is_valid_drivename($bootdisk);
3117
3118     return undef if !$conf->{$bootdisk};
3119
3120     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3121     return undef if !defined($drive);
3122
3123     return undef if drive_is_cdrom($drive);
3124
3125     my $volid = $drive->{file};
3126     return undef if !$volid;
3127
3128     return $drive->{size};
3129 }
3130
3131 our $vmstatus_return_properties = {
3132     vmid => get_standard_option('pve-vmid'),
3133     status => {
3134         description => "Qemu process status.",
3135         type => 'string',
3136         enum => ['stopped', 'running'],
3137     },
3138     maxmem => {
3139         description => "Maximum memory in bytes.",
3140         type => 'integer',
3141         optional => 1,
3142         renderer => 'bytes',
3143     },
3144     maxdisk => {
3145         description => "Root disk size in bytes.",
3146         type => 'integer',
3147         optional => 1,
3148         renderer => 'bytes',
3149     },
3150     name => {
3151         description => "VM name.",
3152         type => 'string',
3153         optional => 1,
3154     },
3155     qmpstatus => {
3156         description => "Qemu QMP agent status.",
3157         type => 'string',
3158         optional => 1,
3159     },
3160     pid => {
3161         description => "PID of running qemu process.",
3162         type => 'integer',
3163         optional => 1,
3164     },
3165     uptime => {
3166         description => "Uptime.",
3167         type => 'integer',
3168         optional => 1,
3169         renderer => 'duration',
3170     },
3171     cpus => {
3172         description => "Maximum usable CPUs.",
3173         type => 'number',
3174         optional => 1,
3175     },
3176     lock => {
3177         description => "The current config lock, if any.",
3178         type => 'string',
3179         optional => 1,
3180     }
3181 };
3182
3183 my $last_proc_pid_stat;
3184
3185 # get VM status information
3186 # This must be fast and should not block ($full == false)
3187 # We only query KVM using QMP if $full == true (this can be slow)
3188 sub vmstatus {
3189     my ($opt_vmid, $full) = @_;
3190
3191     my $res = {};
3192
3193     my $storecfg = PVE::Storage::config();
3194
3195     my $list = vzlist();
3196     my $defaults = load_defaults();
3197
3198     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3199
3200     my $cpucount = $cpuinfo->{cpus} || 1;
3201
3202     foreach my $vmid (keys %$list) {
3203         next if $opt_vmid && ($vmid ne $opt_vmid);
3204
3205         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3206         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3207
3208         my $d = { vmid => $vmid };
3209         $d->{pid} = $list->{$vmid}->{pid};
3210
3211         # fixme: better status?
3212         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3213
3214         my $size = disksize($storecfg, $conf);
3215         if (defined($size)) {
3216             $d->{disk} = 0; # no info available
3217             $d->{maxdisk} = $size;
3218         } else {
3219             $d->{disk} = 0;
3220             $d->{maxdisk} = 0;
3221         }
3222
3223         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3224             * ($conf->{cores} || $defaults->{cores});
3225         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3226         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3227
3228         $d->{name} = $conf->{name} || "VM $vmid";
3229         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3230             : $defaults->{memory}*(1024*1024);
3231
3232         if ($conf->{balloon}) {
3233             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3234             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3235                 : $defaults->{shares};
3236         }
3237
3238         $d->{uptime} = 0;
3239         $d->{cpu} = 0;
3240         $d->{mem} = 0;
3241
3242         $d->{netout} = 0;
3243         $d->{netin} = 0;
3244
3245         $d->{diskread} = 0;
3246         $d->{diskwrite} = 0;
3247
3248         $d->{template} = PVE::QemuConfig->is_template($conf);
3249
3250         $d->{serial} = 1 if conf_has_serial($conf);
3251         $d->{lock} = $conf->{lock} if $conf->{lock};
3252
3253         $res->{$vmid} = $d;
3254     }
3255
3256     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3257     foreach my $dev (keys %$netdev) {
3258         next if $dev !~ m/^tap([1-9]\d*)i/;
3259         my $vmid = $1;
3260         my $d = $res->{$vmid};
3261         next if !$d;
3262
3263         $d->{netout} += $netdev->{$dev}->{receive};
3264         $d->{netin} += $netdev->{$dev}->{transmit};
3265
3266         if ($full) {
3267             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3268             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3269         }
3270
3271     }
3272
3273     my $ctime = gettimeofday;
3274
3275     foreach my $vmid (keys %$list) {
3276
3277         my $d = $res->{$vmid};
3278         my $pid = $d->{pid};
3279         next if !$pid;
3280
3281         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3282         next if !$pstat; # not running
3283
3284         my $used = $pstat->{utime} + $pstat->{stime};
3285
3286         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3287
3288         if ($pstat->{vsize}) {
3289             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3290         }
3291
3292         my $old = $last_proc_pid_stat->{$pid};
3293         if (!$old) {
3294             $last_proc_pid_stat->{$pid} = {
3295                 time => $ctime,
3296                 used => $used,
3297                 cpu => 0,
3298             };
3299             next;
3300         }
3301
3302         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3303
3304         if ($dtime > 1000) {
3305             my $dutime = $used -  $old->{used};
3306
3307             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3308             $last_proc_pid_stat->{$pid} = {
3309                 time => $ctime,
3310                 used => $used,
3311                 cpu => $d->{cpu},
3312             };
3313         } else {
3314             $d->{cpu} = $old->{cpu};
3315         }
3316     }
3317
3318     return $res if !$full;
3319
3320     my $qmpclient = PVE::QMPClient->new();
3321
3322     my $ballooncb = sub {
3323         my ($vmid, $resp) = @_;
3324
3325         my $info = $resp->{'return'};
3326         return if !$info->{max_mem};
3327
3328         my $d = $res->{$vmid};
3329
3330         # use memory assigned to VM
3331         $d->{maxmem} = $info->{max_mem};
3332         $d->{balloon} = $info->{actual};
3333
3334         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3335             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3336             $d->{freemem} = $info->{free_mem};
3337         }
3338
3339         $d->{ballooninfo} = $info;
3340     };
3341
3342     my $blockstatscb = sub {
3343         my ($vmid, $resp) = @_;
3344         my $data = $resp->{'return'} || [];
3345         my $totalrdbytes = 0;
3346         my $totalwrbytes = 0;
3347
3348         for my $blockstat (@$data) {
3349             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3350             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3351
3352             $blockstat->{device} =~ s/drive-//;
3353             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3354         }
3355         $res->{$vmid}->{diskread} = $totalrdbytes;
3356         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3357     };
3358
3359     my $statuscb = sub {
3360         my ($vmid, $resp) = @_;
3361
3362         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3363         # this fails if ballon driver is not loaded, so this must be
3364         # the last commnand (following command are aborted if this fails).
3365         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3366
3367         my $status = 'unknown';
3368         if (!defined($status = $resp->{'return'}->{status})) {
3369             warn "unable to get VM status\n";
3370             return;
3371         }
3372
3373         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3374     };
3375
3376     foreach my $vmid (keys %$list) {
3377         next if $opt_vmid && ($vmid ne $opt_vmid);
3378         next if !$res->{$vmid}->{pid}; # not running
3379         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3380     }
3381
3382     $qmpclient->queue_execute(undef, 2);
3383
3384     foreach my $vmid (keys %$list) {
3385         next if $opt_vmid && ($vmid ne $opt_vmid);
3386         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3387     }
3388
3389     return $res;
3390 }
3391
3392 sub foreach_drive {
3393     my ($conf, $func, @param) = @_;
3394
3395     foreach my $ds (valid_drive_names()) {
3396         next if !defined($conf->{$ds});
3397
3398         my $drive = parse_drive($ds, $conf->{$ds});
3399         next if !$drive;
3400
3401         &$func($ds, $drive, @param);
3402     }
3403 }
3404
3405 sub foreach_volid {
3406     my ($conf, $func, @param) = @_;
3407
3408     my $volhash = {};
3409
3410     my $test_volid = sub {
3411         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3412
3413         return if !$volid;
3414
3415         $volhash->{$volid}->{cdrom} //= 1;
3416         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3417
3418         $volhash->{$volid}->{replicate} //= 0;
3419         $volhash->{$volid}->{replicate} = 1 if $replicate;
3420
3421         $volhash->{$volid}->{shared} //= 0;
3422         $volhash->{$volid}->{shared} = 1 if $shared;
3423
3424         $volhash->{$volid}->{referenced_in_config} //= 0;
3425         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3426
3427         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3428             if defined($snapname);
3429         $volhash->{$volid}->{size} = $size if $size;
3430     };
3431
3432     foreach_drive($conf, sub {
3433         my ($ds, $drive) = @_;
3434         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3435     });
3436
3437     foreach my $snapname (keys %{$conf->{snapshots}}) {
3438         my $snap = $conf->{snapshots}->{$snapname};
3439         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3440         foreach_drive($snap, sub {
3441             my ($ds, $drive) = @_;
3442             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3443         });
3444     }
3445
3446     foreach my $volid (keys %$volhash) {
3447         &$func($volid, $volhash->{$volid}, @param);
3448     }
3449 }
3450
3451 sub conf_has_serial {
3452     my ($conf) = @_;
3453
3454     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3455         if ($conf->{"serial$i"}) {
3456             return 1;
3457         }
3458     }
3459
3460     return 0;
3461 }
3462
3463 sub conf_has_audio {
3464     my ($conf, $id) = @_;
3465
3466     $id //= 0;
3467     my $audio = $conf->{"audio$id"};
3468     return undef if !defined($audio);
3469
3470     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3471     my $audiodriver = $audioproperties->{driver} // 'spice';
3472
3473     return {
3474         dev => $audioproperties->{device},
3475         dev_id => "audiodev$id",
3476         backend => $audiodriver,
3477         backend_id => "$audiodriver-backend${id}",
3478     };
3479 }
3480
3481 sub vga_conf_has_spice {
3482     my ($vga) = @_;
3483
3484     my $vgaconf = parse_vga($vga);
3485     my $vgatype = $vgaconf->{type};
3486     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3487
3488     return $1 || 1;
3489 }
3490
3491 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3492 sub get_host_arch() {
3493     $host_arch = (POSIX::uname())[4] if !$host_arch;
3494     return $host_arch;
3495 }
3496
3497 sub is_native($) {
3498     my ($arch) = @_;
3499     return get_host_arch() eq $arch;
3500 }
3501
3502 my $default_machines = {
3503     x86_64 => 'pc',
3504     aarch64 => 'virt',
3505 };
3506
3507 sub get_basic_machine_info {
3508     my ($conf, $forcemachine) = @_;
3509
3510     my $arch = $conf->{arch} // get_host_arch();
3511     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3512     return ($arch, $machine);
3513 }
3514
3515 sub get_ovmf_files($) {
3516     my ($arch) = @_;
3517
3518     my $ovmf = $OVMF->{$arch}
3519         or die "no OVMF images known for architecture '$arch'\n";
3520
3521     return @$ovmf;
3522 }
3523
3524 my $Arch2Qemu = {
3525     aarch64 => '/usr/bin/qemu-system-aarch64',
3526     x86_64 => '/usr/bin/qemu-system-x86_64',
3527 };
3528 sub get_command_for_arch($) {
3529     my ($arch) = @_;
3530     return '/usr/bin/kvm' if is_native($arch);
3531
3532     my $cmd = $Arch2Qemu->{$arch}
3533         or die "don't know how to emulate architecture '$arch'\n";
3534     return $cmd;
3535 }
3536
3537 sub get_cpu_options {
3538     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3539
3540     my $cpuFlags = [];
3541     my $ostype = $conf->{ostype};
3542
3543     my $cpu = $kvm ? "kvm64" : "qemu64";
3544     if ($arch eq 'aarch64') {
3545         $cpu = 'cortex-a57';
3546     }
3547     my $hv_vendor_id;
3548     if (my $cputype = $conf->{cpu}) {
3549         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3550             or die "Cannot parse cpu description: $cputype\n";
3551         $cpu = $cpuconf->{cputype};
3552         $kvm_off = 1 if $cpuconf->{hidden};
3553         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3554
3555         if (defined(my $flags = $cpuconf->{flags})) {
3556             push @$cpuFlags, split(";", $flags);
3557         }
3558     }
3559
3560     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3561
3562     push @$cpuFlags , '-x2apic'
3563         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3564
3565     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3566
3567     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3568
3569     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3570
3571         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3572         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3573     }
3574
3575     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3576
3577     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3578
3579     push @$cpuFlags, 'kvm=off' if $kvm_off;
3580
3581     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3582         push @$cpuFlags, "vendor=${cpu_vendor}"
3583             if $cpu_vendor ne 'default';
3584     } elsif ($arch ne 'aarch64') {
3585         die "internal error"; # should not happen
3586     }
3587
3588     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3589
3590     return ('-cpu', $cpu);
3591 }
3592
3593 sub config_to_command {
3594     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3595
3596     my $cmd = [];
3597     my $globalFlags = [];
3598     my $machineFlags = [];
3599     my $rtcFlags = [];
3600     my $devices = [];
3601     my $pciaddr = '';
3602     my $bridges = {};
3603     my $vernum = 0; # unknown
3604     my $ostype = $conf->{ostype};
3605     my $winversion = windows_version($ostype);
3606     my $kvm = $conf->{kvm};
3607
3608     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3609     my $kvm_binary = get_command_for_arch($arch);
3610     my $kvmver = kvm_user_version($kvm_binary);
3611     $kvm //= 1 if is_native($arch);
3612
3613     if ($kvm) {
3614         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3615             if !defined kvm_version();
3616     }
3617
3618     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3619         $vernum = $1*1000000+$2*1000;
3620     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3621         $vernum = $1*1000000+$2*1000+$3;
3622     }
3623
3624     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3625
3626     my $have_ovz = -f '/proc/vz/vestat';
3627
3628     my $q35 = machine_type_is_q35($conf);
3629     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3630     my $use_old_bios_files = undef;
3631     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3632
3633     my $cpuunits = defined($conf->{cpuunits}) ?
3634             $conf->{cpuunits} : $defaults->{cpuunits};
3635
3636     push @$cmd, $kvm_binary;
3637
3638     push @$cmd, '-id', $vmid;
3639
3640     my $vmname = $conf->{name} || "vm$vmid";
3641
3642     push @$cmd, '-name', $vmname;
3643
3644     my $use_virtio = 0;
3645
3646     my $qmpsocket = qmp_socket($vmid);
3647     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3648     push @$cmd, '-mon', "chardev=qmp,mode=control";
3649
3650     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3651         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3652         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3653     }
3654
3655     push @$cmd, '-pidfile' , pidfile_name($vmid);
3656
3657     push @$cmd, '-daemonize';
3658
3659     if ($conf->{smbios1}) {
3660         my $smbios_conf = parse_smbios1($conf->{smbios1});
3661         if ($smbios_conf->{base64}) {
3662             # Do not pass base64 flag to qemu
3663             delete $smbios_conf->{base64};
3664             my $smbios_string = "";
3665             foreach my $key (keys %$smbios_conf) {
3666                 my $value;
3667                 if ($key eq "uuid") {
3668                     $value = $smbios_conf->{uuid}
3669                 } else {
3670                     $value = decode_base64($smbios_conf->{$key});
3671                 }
3672                 # qemu accepts any binary data, only commas need escaping by double comma
3673                 $value =~ s/,/,,/g;
3674                 $smbios_string .= "," . $key . "=" . $value if $value;
3675             }
3676             push @$cmd, '-smbios', "type=1" . $smbios_string;
3677         } else {
3678             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3679         }
3680     }
3681
3682     if ($conf->{vmgenid}) {
3683         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3684     }
3685
3686     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3687     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3688         die "uefi base image not found\n" if ! -f $ovmf_code;
3689
3690         my $path;
3691         my $format;
3692         if (my $efidisk = $conf->{efidisk0}) {
3693             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3694             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3695             $format = $d->{format};
3696             if ($storeid) {
3697                 $path = PVE::Storage::path($storecfg, $d->{file});
3698                 if (!defined($format)) {
3699                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3700                     $format = qemu_img_format($scfg, $volname);
3701                 }
3702             } else {
3703                 $path = $d->{file};
3704                 die "efidisk format must be specified\n"
3705                     if !defined($format);
3706             }
3707         } else {
3708             warn "no efidisk configured! Using temporary efivars disk.\n";
3709             $path = "/tmp/$vmid-ovmf.fd";
3710             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3711             $format = 'raw';
3712         }
3713
3714         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3715         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3716     }
3717
3718     # load q35 config
3719     if ($q35) {
3720         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3721         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3722             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3723         } else {
3724             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3725         }
3726     }
3727
3728     # add usb controllers
3729     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3730     push @$devices, @usbcontrollers if @usbcontrollers;
3731     my $vga = parse_vga($conf->{vga});
3732
3733     my $qxlnum = vga_conf_has_spice($conf->{vga});
3734     $vga->{type} = 'qxl' if $qxlnum;
3735
3736     if (!$vga->{type}) {
3737         if ($arch eq 'aarch64') {
3738             $vga->{type} = 'virtio';
3739         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3740             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3741         } else {
3742             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3743         }
3744     }
3745
3746     # enable absolute mouse coordinates (needed by vnc)
3747     my $tablet;
3748     if (defined($conf->{tablet})) {
3749         $tablet = $conf->{tablet};
3750     } else {
3751         $tablet = $defaults->{tablet};
3752         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3753         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3754     }
3755
3756     if ($tablet) {
3757         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3758         my $kbd = print_keyboarddevice_full($conf, $arch);
3759         push @$devices, '-device', $kbd if defined($kbd);
3760     }
3761
3762     my $kvm_off = 0;
3763     my $gpu_passthrough;
3764
3765     # host pci devices
3766     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3767         my $id = "hostpci$i";
3768         my $d = parse_hostpci($conf->{$id});
3769         next if !$d;
3770
3771         if (my $pcie = $d->{pcie}) {
3772             die "q35 machine model is not enabled" if !$q35;
3773             # win7 wants to have the pcie devices directly on the pcie bus
3774             # instead of in the root port
3775             if ($winversion == 7) {
3776                 $pciaddr = print_pcie_addr("${id}bus0");
3777             } else {
3778                 # add more root ports if needed, 4 are present by default
3779                 # by pve-q35 cfgs, rest added here on demand.
3780                 if ($i > 3) {
3781                     push @$devices, '-device', print_pcie_root_port($i);
3782                 }
3783                 $pciaddr = print_pcie_addr($id);
3784             }
3785         } else {
3786             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3787         }
3788
3789         my $xvga = '';
3790         if ($d->{'x-vga'}) {
3791             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3792             $kvm_off = 1;
3793             $vga->{type} = 'none' if !defined($conf->{vga});
3794             $gpu_passthrough = 1;
3795         }
3796
3797         my $pcidevices = $d->{pciid};
3798         my $multifunction = 1 if @$pcidevices > 1;
3799
3800         my $sysfspath;
3801         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3802             my $pci_id = $pcidevices->[0]->{id};
3803             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3804             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3805         } elsif ($d->{mdev}) {
3806             warn "ignoring mediated device '$id' with multifunction device\n";
3807         }
3808
3809         my $j=0;
3810         foreach my $pcidevice (@$pcidevices) {
3811             my $devicestr = "vfio-pci";
3812
3813             if ($sysfspath) {
3814                 $devicestr .= ",sysfsdev=$sysfspath";
3815             } else {
3816                 $devicestr .= ",host=$pcidevice->{id}";
3817             }
3818
3819             my $mf_addr = $multifunction ? ".$j" : '';
3820             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3821
3822             if ($j == 0) {
3823                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3824                 $devicestr .= "$xvga";
3825                 $devicestr .= ",multifunction=on" if $multifunction;
3826                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3827             }
3828
3829             push @$devices, '-device', $devicestr;
3830             $j++;
3831         }
3832     }
3833
3834     # usb devices
3835     my $usb_dev_features = {};
3836     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3837
3838     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3839     push @$devices, @usbdevices if @usbdevices;
3840     # serial devices
3841     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3842         if (my $path = $conf->{"serial$i"}) {
3843             if ($path eq 'socket') {
3844                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3845                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3846                 # On aarch64, serial0 is the UART device. Qemu only allows
3847                 # connecting UART devices via the '-serial' command line, as
3848                 # the device has a fixed slot on the hardware...
3849                 if ($arch eq 'aarch64' && $i == 0) {
3850                     push @$devices, '-serial', "chardev:serial$i";
3851                 } else {
3852                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3853                 }
3854             } else {
3855                 die "no such serial device\n" if ! -c $path;
3856                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3857                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3858             }
3859         }
3860     }
3861
3862     # parallel devices
3863     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3864         if (my $path = $conf->{"parallel$i"}) {
3865             die "no such parallel device\n" if ! -c $path;
3866             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3867             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3868             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3869         }
3870     }
3871
3872     if (my $audio = conf_has_audio($conf)) {
3873
3874         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3875
3876         my $id = $audio->{dev_id};
3877         if ($audio->{dev} eq 'AC97') {
3878             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3879         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3880             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3881             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3882             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3883         } else {
3884             die "unkown audio device '$audio->{dev}', implement me!";
3885         }
3886
3887         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3888     }
3889
3890     my $sockets = 1;
3891     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3892     $sockets = $conf->{sockets} if  $conf->{sockets};
3893
3894     my $cores = $conf->{cores} || 1;
3895
3896     my $maxcpus = $sockets * $cores;
3897
3898     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3899
3900     my $allowed_vcpus = $cpuinfo->{cpus};
3901
3902     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3903         if ($allowed_vcpus < $maxcpus);
3904
3905     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3906
3907         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3908         for (my $i = 2; $i <= $vcpus; $i++)  {
3909             my $cpustr = print_cpu_device($conf,$i);
3910             push @$cmd, '-device', $cpustr;
3911         }
3912
3913     } else {
3914
3915         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3916     }
3917     push @$cmd, '-nodefaults';
3918
3919     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3920
3921     my $bootindex_hash = {};
3922     my $i = 1;
3923     foreach my $o (split(//, $bootorder)) {
3924         $bootindex_hash->{$o} = $i*100;
3925         $i++;
3926     }
3927
3928     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3929
3930     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3931
3932     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3933
3934     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3935         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3936         my $socket = vnc_socket($vmid);
3937         push @$cmd,  '-vnc', "unix:$socket,password";
3938     } else {
3939         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3940         push @$cmd, '-nographic';
3941     }
3942
3943     # time drift fix
3944     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3945
3946     my $useLocaltime = $conf->{localtime};
3947
3948     if ($winversion >= 5) { # windows
3949         $useLocaltime = 1 if !defined($conf->{localtime});
3950
3951         # use time drift fix when acpi is enabled
3952         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3953             $tdf = 1 if !defined($conf->{tdf});
3954         }
3955     }
3956
3957     if ($winversion >= 6) {
3958         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3959         push @$cmd, '-no-hpet';
3960     }
3961
3962     push @$rtcFlags, 'driftfix=slew' if $tdf;
3963
3964     if (!$kvm) {
3965         push @$machineFlags, 'accel=tcg';
3966     }
3967
3968     if ($machine_type) {
3969         push @$machineFlags, "type=${machine_type}";
3970     }
3971
3972     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3973         push @$rtcFlags, "base=$conf->{startdate}";
3974     } elsif ($useLocaltime) {
3975         push @$rtcFlags, 'base=localtime';
3976     }
3977
3978     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3979
3980     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3981
3982     push @$cmd, '-S' if $conf->{freeze};
3983
3984     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3985
3986     if (parse_guest_agent($conf)->{enabled}) {
3987         my $qgasocket = qmp_socket($vmid, 1);
3988         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3989         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3990         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3991         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3992     }
3993
3994     my $spice_port;
3995
3996     if ($qxlnum) {
3997         if ($qxlnum > 1) {
3998             if ($winversion){
3999                 for(my $i = 1; $i < $qxlnum; $i++){
4000                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
4001                 }
4002             } else {
4003                 # assume other OS works like Linux
4004                 my ($ram, $vram) = ("134217728", "67108864");
4005                 if ($vga->{memory}) {
4006                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4007                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4008                 }
4009                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4010                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4011             }
4012         }
4013
4014         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4015
4016         my $nodename = PVE::INotify::nodename();
4017         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4018         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4019         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4020         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4021         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4022
4023         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4024         if ($spice_enhancement->{foldersharing}) {
4025             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4026             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4027         }
4028
4029         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4030         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4031         push @$devices, '-spice', "$spice_opts";
4032
4033         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4034         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4035         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4036
4037     }
4038
4039     # enable balloon by default, unless explicitly disabled
4040     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4041         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4042         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4043     }
4044
4045     if ($conf->{watchdog}) {
4046         my $wdopts = parse_watchdog($conf->{watchdog});
4047         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4048         my $watchdog = $wdopts->{model} || 'i6300esb';
4049         push @$devices, '-device', "$watchdog$pciaddr";
4050         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4051     }
4052
4053     my $vollist = [];
4054     my $scsicontroller = {};
4055     my $ahcicontroller = {};
4056     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4057
4058     # Add iscsi initiator name if available
4059     if (my $initiator = get_initiator_name()) {
4060         push @$devices, '-iscsi', "initiator-name=$initiator";
4061     }
4062
4063     foreach_drive($conf, sub {
4064         my ($ds, $drive) = @_;
4065
4066         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4067             push @$vollist, $drive->{file};
4068         }
4069
4070         # ignore efidisk here, already added in bios/fw handling code above
4071         return if $drive->{interface} eq 'efidisk';
4072
4073         $use_virtio = 1 if $ds =~ m/^virtio/;
4074
4075         if (drive_is_cdrom ($drive)) {
4076             if ($bootindex_hash->{d}) {
4077                 $drive->{bootindex} = $bootindex_hash->{d};
4078                 $bootindex_hash->{d} += 1;
4079             }
4080         } else {
4081             if ($bootindex_hash->{c}) {
4082                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4083                 $bootindex_hash->{c} += 1;
4084             }
4085         }
4086
4087         if($drive->{interface} eq 'virtio'){
4088            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4089         }
4090
4091         if ($drive->{interface} eq 'scsi') {
4092
4093             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4094
4095             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4096             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4097
4098             my $iothread = '';
4099             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4100                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4101                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4102             } elsif ($drive->{iothread}) {
4103                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4104             }
4105
4106             my $queues = '';
4107             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4108                 $queues = ",num_queues=$drive->{queues}";
4109             }
4110
4111             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4112             $scsicontroller->{$controller}=1;
4113         }
4114
4115         if ($drive->{interface} eq 'sata') {
4116            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4117            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4118            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4119            $ahcicontroller->{$controller}=1;
4120         }
4121
4122         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4123         push @$devices, '-drive',$drive_cmd;
4124         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4125     });
4126
4127     for (my $i = 0; $i < $MAX_NETS; $i++) {
4128          next if !$conf->{"net$i"};
4129          my $d = parse_net($conf->{"net$i"});
4130          next if !$d;
4131
4132          $use_virtio = 1 if $d->{model} eq 'virtio';
4133
4134          if ($bootindex_hash->{n}) {
4135             $d->{bootindex} = $bootindex_hash->{n};
4136             $bootindex_hash->{n} += 1;
4137          }
4138
4139          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4140          push @$devices, '-netdev', $netdevfull;
4141
4142          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4143          push @$devices, '-device', $netdevicefull;
4144     }
4145
4146     if ($conf->{ivshmem}) {
4147         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4148
4149         my $bus;
4150         if ($q35) {
4151             $bus = print_pcie_addr("ivshmem");
4152         } else {
4153             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4154         }
4155
4156         my $ivshmem_name = $ivshmem->{name} // $vmid;
4157         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4158
4159         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4160         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4161     }
4162
4163     if (!$q35) {
4164         # add pci bridges
4165         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4166            $bridges->{1} = 1;
4167            $bridges->{2} = 1;
4168         }
4169
4170         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4171
4172         while (my ($k, $v) = each %$bridges) {
4173             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4174             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4175         }
4176     }
4177
4178     push @$cmd, @$devices;
4179     push @$cmd, '-rtc', join(',', @$rtcFlags)
4180         if scalar(@$rtcFlags);
4181     push @$cmd, '-machine', join(',', @$machineFlags)
4182         if scalar(@$machineFlags);
4183     push @$cmd, '-global', join(',', @$globalFlags)
4184         if scalar(@$globalFlags);
4185
4186     if (my $vmstate = $conf->{vmstate}) {
4187         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4188         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4189         push @$cmd, '-loadstate', $statepath;
4190     }
4191
4192     # add custom args
4193     if ($conf->{args}) {
4194         my $aa = PVE::Tools::split_args($conf->{args});
4195         push @$cmd, @$aa;
4196     }
4197
4198     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4199 }
4200
4201 sub vnc_socket {
4202     my ($vmid) = @_;
4203     return "${var_run_tmpdir}/$vmid.vnc";
4204 }
4205
4206 sub spice_port {
4207     my ($vmid) = @_;
4208
4209     my $res = vm_mon_cmd($vmid, 'query-spice');
4210
4211     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4212 }
4213
4214 sub qmp_socket {
4215     my ($vmid, $qga, $name) = @_;
4216     my $sockettype = $qga ? 'qga' : 'qmp';
4217     my $ext = $name ? '-'.$name : '';
4218     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4219 }
4220
4221 sub pidfile_name {
4222     my ($vmid) = @_;
4223     return "${var_run_tmpdir}/$vmid.pid";
4224 }
4225
4226 sub vm_devices_list {
4227     my ($vmid) = @_;
4228
4229     my $res = vm_mon_cmd($vmid, 'query-pci');
4230     my $devices_to_check = [];
4231     my $devices = {};
4232     foreach my $pcibus (@$res) {
4233         push @$devices_to_check, @{$pcibus->{devices}},
4234     }
4235
4236     while (@$devices_to_check) {
4237         my $to_check = [];
4238         for my $d (@$devices_to_check) {
4239             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4240             next if !$d->{'pci_bridge'};
4241
4242             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4243             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4244         }
4245         $devices_to_check = $to_check;
4246     }
4247
4248     my $resblock = vm_mon_cmd($vmid, 'query-block');
4249     foreach my $block (@$resblock) {
4250         if($block->{device} =~ m/^drive-(\S+)/){
4251                 $devices->{$1} = 1;
4252         }
4253     }
4254
4255     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4256     foreach my $mice (@$resmice) {
4257         if ($mice->{name} eq 'QEMU HID Tablet') {
4258             $devices->{tablet} = 1;
4259             last;
4260         }
4261     }
4262
4263     # for usb devices there is no query-usb
4264     # but we can iterate over the entries in
4265     # qom-list path=/machine/peripheral
4266     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4267     foreach my $per (@$resperipheral) {
4268         if ($per->{name} =~ m/^usb\d+$/) {
4269             $devices->{$per->{name}} = 1;
4270         }
4271     }
4272
4273     return $devices;
4274 }
4275
4276 sub vm_deviceplug {
4277     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4278
4279     my $q35 = machine_type_is_q35($conf);
4280
4281     my $devices_list = vm_devices_list($vmid);
4282     return 1 if defined($devices_list->{$deviceid});
4283
4284     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4285
4286     if ($deviceid eq 'tablet') {
4287
4288         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4289
4290     } elsif ($deviceid eq 'keyboard') {
4291
4292         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4293
4294     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4295
4296         die "usb hotplug currently not reliable\n";
4297         # since we can't reliably hot unplug all added usb devices
4298         # and usb passthrough disables live migration
4299         # we disable usb hotplugging for now
4300         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4301
4302     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4303
4304         qemu_iothread_add($vmid, $deviceid, $device);
4305
4306         qemu_driveadd($storecfg, $vmid, $device);
4307         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4308
4309         qemu_deviceadd($vmid, $devicefull);
4310         eval { qemu_deviceaddverify($vmid, $deviceid); };
4311         if (my $err = $@) {
4312             eval { qemu_drivedel($vmid, $deviceid); };
4313             warn $@ if $@;
4314             die $err;
4315         }
4316
4317     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4318
4319
4320         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4321         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4322         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4323
4324         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4325
4326         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4327             qemu_iothread_add($vmid, $deviceid, $device);
4328             $devicefull .= ",iothread=iothread-$deviceid";
4329         }
4330
4331         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4332             $devicefull .= ",num_queues=$device->{queues}";
4333         }
4334
4335         qemu_deviceadd($vmid, $devicefull);
4336         qemu_deviceaddverify($vmid, $deviceid);
4337
4338     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4339
4340         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4341         qemu_driveadd($storecfg, $vmid, $device);
4342
4343         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4344         eval { qemu_deviceadd($vmid, $devicefull); };
4345         if (my $err = $@) {
4346             eval { qemu_drivedel($vmid, $deviceid); };
4347             warn $@ if $@;
4348             die $err;
4349         }
4350
4351     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4352
4353         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4354
4355         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4356         my $use_old_bios_files = undef;
4357         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4358
4359         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4360         qemu_deviceadd($vmid, $netdevicefull);
4361         eval {
4362             qemu_deviceaddverify($vmid, $deviceid);
4363             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4364         };
4365         if (my $err = $@) {
4366             eval { qemu_netdevdel($vmid, $deviceid); };
4367             warn $@ if $@;
4368             die $err;
4369         }
4370
4371     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4372
4373         my $bridgeid = $2;
4374         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4375         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4376
4377         qemu_deviceadd($vmid, $devicefull);
4378         qemu_deviceaddverify($vmid, $deviceid);
4379
4380     } else {
4381         die "can't hotplug device '$deviceid'\n";
4382     }
4383
4384     return 1;
4385 }
4386
4387 # fixme: this should raise exceptions on error!
4388 sub vm_deviceunplug {
4389     my ($vmid, $conf, $deviceid) = @_;
4390
4391     my $devices_list = vm_devices_list($vmid);
4392     return 1 if !defined($devices_list->{$deviceid});
4393
4394     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4395
4396     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4397
4398         qemu_devicedel($vmid, $deviceid);
4399
4400     } elsif ($deviceid =~ m/^usb\d+$/) {
4401
4402         die "usb hotplug currently not reliable\n";
4403         # when unplugging usb devices this way,
4404         # there may be remaining usb controllers/hubs
4405         # so we disable it for now
4406         qemu_devicedel($vmid, $deviceid);
4407         qemu_devicedelverify($vmid, $deviceid);
4408
4409     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4410
4411         qemu_devicedel($vmid, $deviceid);
4412         qemu_devicedelverify($vmid, $deviceid);
4413         qemu_drivedel($vmid, $deviceid);
4414         qemu_iothread_del($conf, $vmid, $deviceid);
4415
4416     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4417
4418         qemu_devicedel($vmid, $deviceid);
4419         qemu_devicedelverify($vmid, $deviceid);
4420         qemu_iothread_del($conf, $vmid, $deviceid);
4421
4422     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4423
4424         qemu_devicedel($vmid, $deviceid);
4425         qemu_drivedel($vmid, $deviceid);
4426         qemu_deletescsihw($conf, $vmid, $deviceid);
4427
4428     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4429
4430         qemu_devicedel($vmid, $deviceid);
4431         qemu_devicedelverify($vmid, $deviceid);
4432         qemu_netdevdel($vmid, $deviceid);
4433
4434     } else {
4435         die "can't unplug device '$deviceid'\n";
4436     }
4437
4438     return 1;
4439 }
4440
4441 sub qemu_deviceadd {
4442     my ($vmid, $devicefull) = @_;
4443
4444     $devicefull = "driver=".$devicefull;
4445     my %options =  split(/[=,]/, $devicefull);
4446
4447     vm_mon_cmd($vmid, "device_add" , %options);
4448 }
4449
4450 sub qemu_devicedel {
4451     my ($vmid, $deviceid) = @_;
4452
4453     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4454 }
4455
4456 sub qemu_iothread_add {
4457     my($vmid, $deviceid, $device) = @_;
4458
4459     if ($device->{iothread}) {
4460         my $iothreads = vm_iothreads_list($vmid);
4461         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4462     }
4463 }
4464
4465 sub qemu_iothread_del {
4466     my($conf, $vmid, $deviceid) = @_;
4467
4468     my $confid = $deviceid;
4469     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4470         $confid = 'scsi' . $1;
4471     }
4472     my $device = parse_drive($confid, $conf->{$confid});
4473     if ($device->{iothread}) {
4474         my $iothreads = vm_iothreads_list($vmid);
4475         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4476     }
4477 }
4478
4479 sub qemu_objectadd {
4480     my($vmid, $objectid, $qomtype) = @_;
4481
4482     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4483
4484     return 1;
4485 }
4486
4487 sub qemu_objectdel {
4488     my($vmid, $objectid) = @_;
4489
4490     vm_mon_cmd($vmid, "object-del", id => $objectid);
4491
4492     return 1;
4493 }
4494
4495 sub qemu_driveadd {
4496     my ($storecfg, $vmid, $device) = @_;
4497
4498     my $drive = print_drive_full($storecfg, $vmid, $device);
4499     $drive =~ s/\\/\\\\/g;
4500     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4501
4502     # If the command succeeds qemu prints: "OK"
4503     return 1 if $ret =~ m/OK/s;
4504
4505     die "adding drive failed: $ret\n";
4506 }
4507
4508 sub qemu_drivedel {
4509     my($vmid, $deviceid) = @_;
4510
4511     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4512     $ret =~ s/^\s+//;
4513
4514     return 1 if $ret eq "";
4515
4516     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4517     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4518
4519     die "deleting drive $deviceid failed : $ret\n";
4520 }
4521
4522 sub qemu_deviceaddverify {
4523     my ($vmid, $deviceid) = @_;
4524
4525     for (my $i = 0; $i <= 5; $i++) {
4526          my $devices_list = vm_devices_list($vmid);
4527          return 1 if defined($devices_list->{$deviceid});
4528          sleep 1;
4529     }
4530
4531     die "error on hotplug device '$deviceid'\n";
4532 }
4533
4534
4535 sub qemu_devicedelverify {
4536     my ($vmid, $deviceid) = @_;
4537
4538     # need to verify that the device is correctly removed as device_del
4539     # is async and empty return is not reliable
4540
4541     for (my $i = 0; $i <= 5; $i++) {
4542          my $devices_list = vm_devices_list($vmid);
4543          return 1 if !defined($devices_list->{$deviceid});
4544          sleep 1;
4545     }
4546
4547     die "error on hot-unplugging device '$deviceid'\n";
4548 }
4549
4550 sub qemu_findorcreatescsihw {
4551     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4552
4553     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4554
4555     my $scsihwid="$controller_prefix$controller";
4556     my $devices_list = vm_devices_list($vmid);
4557
4558     if(!defined($devices_list->{$scsihwid})) {
4559         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4560     }
4561
4562     return 1;
4563 }
4564
4565 sub qemu_deletescsihw {
4566     my ($conf, $vmid, $opt) = @_;
4567
4568     my $device = parse_drive($opt, $conf->{$opt});
4569
4570     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4571         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4572         return 1;
4573     }
4574
4575     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4576
4577     my $devices_list = vm_devices_list($vmid);
4578     foreach my $opt (keys %{$devices_list}) {
4579         if (PVE::QemuServer::is_valid_drivename($opt)) {
4580             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4581             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4582                 return 1;
4583             }
4584         }
4585     }
4586
4587     my $scsihwid="scsihw$controller";
4588
4589     vm_deviceunplug($vmid, $conf, $scsihwid);
4590
4591     return 1;
4592 }
4593
4594 sub qemu_add_pci_bridge {
4595     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4596
4597     my $bridges = {};
4598
4599     my $bridgeid;
4600
4601     print_pci_addr($device, $bridges, $arch, $machine_type);
4602
4603     while (my ($k, $v) = each %$bridges) {
4604         $bridgeid = $k;
4605     }
4606     return 1 if !defined($bridgeid) || $bridgeid < 1;
4607
4608     my $bridge = "pci.$bridgeid";
4609     my $devices_list = vm_devices_list($vmid);
4610
4611     if (!defined($devices_list->{$bridge})) {
4612         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4613     }
4614
4615     return 1;
4616 }
4617
4618 sub qemu_set_link_status {
4619     my ($vmid, $device, $up) = @_;
4620
4621     vm_mon_cmd($vmid, "set_link", name => $device,
4622                up => $up ? JSON::true : JSON::false);
4623 }
4624
4625 sub qemu_netdevadd {
4626     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4627
4628     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4629     my %options =  split(/[=,]/, $netdev);
4630
4631     vm_mon_cmd($vmid, "netdev_add",  %options);
4632     return 1;
4633 }
4634
4635 sub qemu_netdevdel {
4636     my ($vmid, $deviceid) = @_;
4637
4638     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4639 }
4640
4641 sub qemu_usb_hotplug {
4642     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4643
4644     return if !$device;
4645
4646     # remove the old one first
4647     vm_deviceunplug($vmid, $conf, $deviceid);
4648
4649     # check if xhci controller is necessary and available
4650     if ($device->{usb3}) {
4651
4652         my $devicelist = vm_devices_list($vmid);
4653
4654         if (!$devicelist->{xhci}) {
4655             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4656             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4657         }
4658     }
4659     my $d = parse_usb_device($device->{host});
4660     $d->{usb3} = $device->{usb3};
4661
4662     # add the new one
4663     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4664 }
4665
4666 sub qemu_cpu_hotplug {
4667     my ($vmid, $conf, $vcpus) = @_;
4668
4669     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4670
4671     my $sockets = 1;
4672     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4673     $sockets = $conf->{sockets} if  $conf->{sockets};
4674     my $cores = $conf->{cores} || 1;
4675     my $maxcpus = $sockets * $cores;
4676
4677     $vcpus = $maxcpus if !$vcpus;
4678
4679     die "you can't add more vcpus than maxcpus\n"
4680         if $vcpus > $maxcpus;
4681
4682     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4683
4684     if ($vcpus < $currentvcpus) {
4685
4686         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4687
4688             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4689                 qemu_devicedel($vmid, "cpu$i");
4690                 my $retry = 0;
4691                 my $currentrunningvcpus = undef;
4692                 while (1) {
4693                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4694                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4695                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4696                     $retry++;
4697                     sleep 1;
4698                 }
4699                 #update conf after each succesfull cpu unplug
4700                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4701                 PVE::QemuConfig->write_config($vmid, $conf);
4702             }
4703         } else {
4704             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4705         }
4706
4707         return;
4708     }
4709
4710     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4711     die "vcpus in running vm does not match its configuration\n"
4712         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4713
4714     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4715
4716         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4717             my $cpustr = print_cpu_device($conf, $i);
4718             qemu_deviceadd($vmid, $cpustr);
4719
4720             my $retry = 0;
4721             my $currentrunningvcpus = undef;
4722             while (1) {
4723                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4724                 last if scalar(@{$currentrunningvcpus}) == $i;
4725                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4726                 sleep 1;
4727                 $retry++;
4728             }
4729             #update conf after each succesfull cpu hotplug
4730             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4731             PVE::QemuConfig->write_config($vmid, $conf);
4732         }
4733     } else {
4734
4735         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4736             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4737         }
4738     }
4739 }
4740
4741 sub qemu_block_set_io_throttle {
4742     my ($vmid, $deviceid,
4743         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4744         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4745         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4746         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4747
4748     return if !check_running($vmid) ;
4749
4750     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4751         bps => int($bps),
4752         bps_rd => int($bps_rd),
4753         bps_wr => int($bps_wr),
4754         iops => int($iops),
4755         iops_rd => int($iops_rd),
4756         iops_wr => int($iops_wr),
4757         bps_max => int($bps_max),
4758         bps_rd_max => int($bps_rd_max),
4759         bps_wr_max => int($bps_wr_max),
4760         iops_max => int($iops_max),
4761         iops_rd_max => int($iops_rd_max),
4762         iops_wr_max => int($iops_wr_max),
4763         bps_max_length => int($bps_max_length),
4764         bps_rd_max_length => int($bps_rd_max_length),
4765         bps_wr_max_length => int($bps_wr_max_length),
4766         iops_max_length => int($iops_max_length),
4767         iops_rd_max_length => int($iops_rd_max_length),
4768         iops_wr_max_length => int($iops_wr_max_length),
4769     );
4770
4771 }
4772
4773 # old code, only used to shutdown old VM after update
4774 sub __read_avail {
4775     my ($fh, $timeout) = @_;
4776
4777     my $sel = new IO::Select;
4778     $sel->add($fh);
4779
4780     my $res = '';
4781     my $buf;
4782
4783     my @ready;
4784     while (scalar (@ready = $sel->can_read($timeout))) {
4785         my $count;
4786         if ($count = $fh->sysread($buf, 8192)) {
4787             if ($buf =~ /^(.*)\(qemu\) $/s) {
4788                 $res .= $1;
4789                 last;
4790             } else {
4791                 $res .= $buf;
4792             }
4793         } else {
4794             if (!defined($count)) {
4795                 die "$!\n";
4796             }
4797             last;
4798         }
4799     }
4800
4801     die "monitor read timeout\n" if !scalar(@ready);
4802
4803     return $res;
4804 }
4805
4806 sub qemu_block_resize {
4807     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4808
4809     my $running = check_running($vmid);
4810
4811     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4812
4813     return if !$running;
4814
4815     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4816
4817 }
4818
4819 sub qemu_volume_snapshot {
4820     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4821
4822     my $running = check_running($vmid);
4823
4824     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4825         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4826     } else {
4827         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4828     }
4829 }
4830
4831 sub qemu_volume_snapshot_delete {
4832     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4833
4834     my $running = check_running($vmid);
4835
4836     if($running) {
4837
4838         $running = undef;
4839         my $conf = PVE::QemuConfig->load_config($vmid);
4840         foreach_drive($conf, sub {
4841             my ($ds, $drive) = @_;
4842             $running = 1 if $drive->{file} eq $volid;
4843         });
4844     }
4845
4846     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4847         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4848     } else {
4849         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4850     }
4851 }
4852
4853 sub set_migration_caps {
4854     my ($vmid) = @_;
4855
4856     my $cap_ref = [];
4857
4858     my $enabled_cap = {
4859         "auto-converge" => 1,
4860         "xbzrle" => 1,
4861         "x-rdma-pin-all" => 0,
4862         "zero-blocks" => 0,
4863         "compress" => 0
4864     };
4865
4866     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4867
4868     for my $supported_capability (@$supported_capabilities) {
4869         push @$cap_ref, {
4870             capability => $supported_capability->{capability},
4871             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4872         };
4873     }
4874
4875     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4876 }
4877
4878 my $fast_plug_option = {
4879     'lock' => 1,
4880     'name' => 1,
4881     'onboot' => 1,
4882     'shares' => 1,
4883     'startup' => 1,
4884     'description' => 1,
4885     'protection' => 1,
4886     'vmstatestorage' => 1,
4887     'hookscript' => 1,
4888 };
4889
4890 # hotplug changes in [PENDING]
4891 # $selection hash can be used to only apply specified options, for
4892 # example: { cores => 1 } (only apply changed 'cores')
4893 # $errors ref is used to return error messages
4894 sub vmconfig_hotplug_pending {
4895     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4896
4897     my $defaults = load_defaults();
4898     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4899
4900     # commit values which do not have any impact on running VM first
4901     # Note: those option cannot raise errors, we we do not care about
4902     # $selection and always apply them.
4903
4904     my $add_error = sub {
4905         my ($opt, $msg) = @_;
4906         $errors->{$opt} = "hotplug problem - $msg";
4907     };
4908
4909     my $changes = 0;
4910     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4911         if ($fast_plug_option->{$opt}) {
4912             $conf->{$opt} = $conf->{pending}->{$opt};
4913             delete $conf->{pending}->{$opt};
4914             $changes = 1;
4915         }
4916     }
4917
4918     if ($changes) {
4919         PVE::QemuConfig->write_config($vmid, $conf);
4920         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4921     }
4922
4923     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4924
4925     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4926     while (my ($opt, $force) = each %$pending_delete_hash) {
4927         next if $selection && !$selection->{$opt};
4928         eval {
4929             if ($opt eq 'hotplug') {
4930                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4931             } elsif ($opt eq 'tablet') {
4932                 die "skip\n" if !$hotplug_features->{usb};
4933                 if ($defaults->{tablet}) {
4934                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4935                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4936                         if $arch eq 'aarch64';
4937                 } else {
4938                     vm_deviceunplug($vmid, $conf, 'tablet');
4939                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4940                 }
4941             } elsif ($opt =~ m/^usb\d+/) {
4942                 die "skip\n";
4943                 # since we cannot reliably hot unplug usb devices
4944                 # we are disabling it
4945                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4946                 vm_deviceunplug($vmid, $conf, $opt);
4947             } elsif ($opt eq 'vcpus') {
4948                 die "skip\n" if !$hotplug_features->{cpu};
4949                 qemu_cpu_hotplug($vmid, $conf, undef);
4950             } elsif ($opt eq 'balloon') {
4951                 # enable balloon device is not hotpluggable
4952                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4953                 # here we reset the ballooning value to memory
4954                 my $balloon = $conf->{memory} || $defaults->{memory};
4955                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4956             } elsif ($fast_plug_option->{$opt}) {
4957                 # do nothing
4958             } elsif ($opt =~ m/^net(\d+)$/) {
4959                 die "skip\n" if !$hotplug_features->{network};
4960                 vm_deviceunplug($vmid, $conf, $opt);
4961             } elsif (is_valid_drivename($opt)) {
4962                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4963                 vm_deviceunplug($vmid, $conf, $opt);
4964                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4965             } elsif ($opt =~ m/^memory$/) {
4966                 die "skip\n" if !$hotplug_features->{memory};
4967                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4968             } elsif ($opt eq 'cpuunits') {
4969                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4970             } elsif ($opt eq 'cpulimit') {
4971                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4972             } else {
4973                 die "skip\n";
4974             }
4975         };
4976         if (my $err = $@) {
4977             &$add_error($opt, $err) if $err ne "skip\n";
4978         } else {
4979             # save new config if hotplug was successful
4980             delete $conf->{$opt};
4981             vmconfig_undelete_pending_option($conf, $opt);
4982             PVE::QemuConfig->write_config($vmid, $conf);
4983             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4984         }
4985     }
4986
4987     my $apply_pending_cloudinit;
4988     $apply_pending_cloudinit = sub {
4989         my ($key, $value) = @_;
4990         $apply_pending_cloudinit = sub {}; # once is enough
4991
4992         my @cloudinit_opts = keys %$confdesc_cloudinit;
4993         foreach my $opt (keys %{$conf->{pending}}) {
4994             next if !grep { $_ eq $opt } @cloudinit_opts;
4995             $conf->{$opt} = delete $conf->{pending}->{$opt};
4996         }
4997
4998         my $new_conf = { %$conf };
4999         $new_conf->{$key} = $value;
5000         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
5001     };
5002
5003     foreach my $opt (keys %{$conf->{pending}}) {
5004         next if $selection && !$selection->{$opt};
5005         my $value = $conf->{pending}->{$opt};
5006         eval {
5007             if ($opt eq 'hotplug') {
5008                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5009             } elsif ($opt eq 'tablet') {
5010                 die "skip\n" if !$hotplug_features->{usb};
5011                 if ($value == 1) {
5012                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5013                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5014                         if $arch eq 'aarch64';
5015                 } elsif ($value == 0) {
5016                     vm_deviceunplug($vmid, $conf, 'tablet');
5017                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5018                 }
5019             } elsif ($opt =~ m/^usb\d+$/) {
5020                 die "skip\n";
5021                 # since we cannot reliably hot unplug usb devices
5022                 # we are disabling it
5023                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5024                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5025                 die "skip\n" if !$d;
5026                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5027             } elsif ($opt eq 'vcpus') {
5028                 die "skip\n" if !$hotplug_features->{cpu};
5029                 qemu_cpu_hotplug($vmid, $conf, $value);
5030             } elsif ($opt eq 'balloon') {
5031                 # enable/disable balloning device is not hotpluggable
5032                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5033                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5034                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5035
5036                 # allow manual ballooning if shares is set to zero
5037                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5038                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5039                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5040                 }
5041             } elsif ($opt =~ m/^net(\d+)$/) {
5042                 # some changes can be done without hotplug
5043                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5044                                     $vmid, $opt, $value, $arch, $machine_type);
5045             } elsif (is_valid_drivename($opt)) {
5046                 # some changes can be done without hotplug
5047                 my $drive = parse_drive($opt, $value);
5048                 if (drive_is_cloudinit($drive)) {
5049                     &$apply_pending_cloudinit($opt, $value);
5050                 }
5051                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5052                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5053             } elsif ($opt =~ m/^memory$/) { #dimms
5054                 die "skip\n" if !$hotplug_features->{memory};
5055                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5056             } elsif ($opt eq 'cpuunits') {
5057                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5058             } elsif ($opt eq 'cpulimit') {
5059                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5060                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5061             } else {
5062                 die "skip\n";  # skip non-hot-pluggable options
5063             }
5064         };
5065         if (my $err = $@) {
5066             &$add_error($opt, $err) if $err ne "skip\n";
5067         } else {
5068             # save new config if hotplug was successful
5069             $conf->{$opt} = $value;
5070             delete $conf->{pending}->{$opt};
5071             PVE::QemuConfig->write_config($vmid, $conf);
5072             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5073         }
5074     }
5075 }
5076
5077 sub try_deallocate_drive {
5078     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5079
5080     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5081         my $volid = $drive->{file};
5082         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5083             my $sid = PVE::Storage::parse_volume_id($volid);
5084             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5085
5086             # check if the disk is really unused
5087             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5088                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5089             PVE::Storage::vdisk_free($storecfg, $volid);
5090             return 1;
5091         } else {
5092             # If vm is not owner of this disk remove from config
5093             return 1;
5094         }
5095     }
5096
5097     return undef;
5098 }
5099
5100 sub vmconfig_delete_or_detach_drive {
5101     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5102
5103     my $drive = parse_drive($opt, $conf->{$opt});
5104
5105     my $rpcenv = PVE::RPCEnvironment::get();
5106     my $authuser = $rpcenv->get_user();
5107
5108     if ($force) {
5109         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5110         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5111     } else {
5112         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5113     }
5114 }
5115
5116 sub vmconfig_apply_pending {
5117     my ($vmid, $conf, $storecfg) = @_;
5118
5119     # cold plug
5120
5121     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5122     while (my ($opt, $force) = each %$pending_delete_hash) {
5123         die "internal error" if $opt =~ m/^unused/;
5124         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5125         if (!defined($conf->{$opt})) {
5126             vmconfig_undelete_pending_option($conf, $opt);
5127             PVE::QemuConfig->write_config($vmid, $conf);
5128         } elsif (is_valid_drivename($opt)) {
5129             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5130             vmconfig_undelete_pending_option($conf, $opt);
5131             delete $conf->{$opt};
5132             PVE::QemuConfig->write_config($vmid, $conf);
5133         } else {
5134             vmconfig_undelete_pending_option($conf, $opt);
5135             delete $conf->{$opt};
5136             PVE::QemuConfig->write_config($vmid, $conf);
5137         }
5138     }
5139
5140     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5141
5142     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5143         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5144
5145         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5146             # skip if nothing changed
5147         } elsif (is_valid_drivename($opt)) {
5148             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5149                 if defined($conf->{$opt});
5150             $conf->{$opt} = $conf->{pending}->{$opt};
5151         } else {
5152             $conf->{$opt} = $conf->{pending}->{$opt};
5153         }
5154
5155         delete $conf->{pending}->{$opt};
5156         PVE::QemuConfig->write_config($vmid, $conf);
5157     }
5158 }
5159
5160 my $safe_num_ne = sub {
5161     my ($a, $b) = @_;
5162
5163     return 0 if !defined($a) && !defined($b);
5164     return 1 if !defined($a);
5165     return 1 if !defined($b);
5166
5167     return $a != $b;
5168 };
5169
5170 my $safe_string_ne = sub {
5171     my ($a, $b) = @_;
5172
5173     return 0 if !defined($a) && !defined($b);
5174     return 1 if !defined($a);
5175     return 1 if !defined($b);
5176
5177     return $a ne $b;
5178 };
5179
5180 sub vmconfig_update_net {
5181     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5182
5183     my $newnet = parse_net($value);
5184
5185     if ($conf->{$opt}) {
5186         my $oldnet = parse_net($conf->{$opt});
5187
5188         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5189             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5190             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5191             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5192
5193             # for non online change, we try to hot-unplug
5194             die "skip\n" if !$hotplug;
5195             vm_deviceunplug($vmid, $conf, $opt);
5196         } else {
5197
5198             die "internal error" if $opt !~ m/net(\d+)/;
5199             my $iface = "tap${vmid}i$1";
5200
5201             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5202                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5203                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5204                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5205                 PVE::Network::tap_unplug($iface);
5206                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5207             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5208                 # Rate can be applied on its own but any change above needs to
5209                 # include the rate in tap_plug since OVS resets everything.
5210                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5211             }
5212
5213             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5214                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5215             }
5216
5217             return 1;
5218         }
5219     }
5220
5221     if ($hotplug) {
5222         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5223     } else {
5224         die "skip\n";
5225     }
5226 }
5227
5228 sub vmconfig_update_disk {
5229     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5230
5231     # fixme: do we need force?
5232
5233     my $drive = parse_drive($opt, $value);
5234
5235     if ($conf->{$opt}) {
5236
5237         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5238
5239             my $media = $drive->{media} || 'disk';
5240             my $oldmedia = $old_drive->{media} || 'disk';
5241             die "unable to change media type\n" if $media ne $oldmedia;
5242
5243             if (!drive_is_cdrom($old_drive)) {
5244
5245                 if ($drive->{file} ne $old_drive->{file}) {
5246
5247                     die "skip\n" if !$hotplug;
5248
5249                     # unplug and register as unused
5250                     vm_deviceunplug($vmid, $conf, $opt);
5251                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5252
5253                 } else {
5254                     # update existing disk
5255
5256                     # skip non hotpluggable value
5257                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5258                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5259                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5260                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5261                         die "skip\n";
5262                     }
5263
5264                     # apply throttle
5265                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5266                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5267                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5268                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5269                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5270                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5271                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5272                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5273                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5274                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5275                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5276                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5277                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5278                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5279                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5280                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5281                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5282                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5283
5284                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5285                                                    ($drive->{mbps} || 0)*1024*1024,
5286                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5287                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5288                                                    $drive->{iops} || 0,
5289                                                    $drive->{iops_rd} || 0,
5290                                                    $drive->{iops_wr} || 0,
5291                                                    ($drive->{mbps_max} || 0)*1024*1024,
5292                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5293                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5294                                                    $drive->{iops_max} || 0,
5295                                                    $drive->{iops_rd_max} || 0,
5296                                                    $drive->{iops_wr_max} || 0,
5297                                                    $drive->{bps_max_length} || 1,
5298                                                    $drive->{bps_rd_max_length} || 1,
5299                                                    $drive->{bps_wr_max_length} || 1,
5300                                                    $drive->{iops_max_length} || 1,
5301                                                    $drive->{iops_rd_max_length} || 1,
5302                                                    $drive->{iops_wr_max_length} || 1);
5303
5304                     }
5305
5306                     return 1;
5307                 }
5308
5309             } else { # cdrom
5310
5311                 if ($drive->{file} eq 'none') {
5312                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5313                     if (drive_is_cloudinit($old_drive)) {
5314                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5315                     }
5316                 } else {
5317                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5318                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5319                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5320                 }
5321
5322                 return 1;
5323             }
5324         }
5325     }
5326
5327     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5328     # hotplug new disks
5329     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5330     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5331 }
5332
5333 sub vm_start {
5334     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5335         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5336
5337     PVE::QemuConfig->lock_config($vmid, sub {
5338         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5339
5340         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5341
5342         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5343
5344         PVE::QemuConfig->check_lock($conf)
5345             if !($skiplock || $is_suspended);
5346
5347         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5348
5349         # clean up leftover reboot request files
5350         eval { clear_reboot_request($vmid); };
5351         warn $@ if $@;
5352
5353         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5354             vmconfig_apply_pending($vmid, $conf, $storecfg);
5355             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5356         }
5357
5358         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5359
5360         my $defaults = load_defaults();
5361
5362         # set environment variable useful inside network script
5363         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5364
5365         my $local_volumes = {};
5366
5367         if ($targetstorage) {
5368             foreach_drive($conf, sub {
5369                 my ($ds, $drive) = @_;
5370
5371                 return if drive_is_cdrom($drive);
5372
5373                 my $volid = $drive->{file};
5374
5375                 return if !$volid;
5376
5377                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5378
5379                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5380                 return if $scfg->{shared};
5381                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5382             });
5383
5384             my $format = undef;
5385
5386             foreach my $opt (sort keys %$local_volumes) {
5387
5388                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5389                 my $drive = parse_drive($opt, $conf->{$opt});
5390
5391                 #if remote storage is specified, use default format
5392                 if ($targetstorage && $targetstorage ne "1") {
5393                     $storeid = $targetstorage;
5394                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5395                     $format = $defFormat;
5396                 } else {
5397                     #else we use same format than original
5398                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5399                     $format = qemu_img_format($scfg, $volid);
5400                 }
5401
5402                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5403                 my $newdrive = $drive;
5404                 $newdrive->{format} = $format;
5405                 $newdrive->{file} = $newvolid;
5406                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5407                 $local_volumes->{$opt} = $drivestr;
5408                 #pass drive to conf for command line
5409                 $conf->{$opt} = $drivestr;
5410             }
5411         }
5412
5413         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5414
5415         if ($is_suspended) {
5416             # enforce machine type on suspended vm to ensure HW compatibility
5417             $forcemachine = $conf->{runningmachine};
5418             print "Resuming suspended VM\n";
5419         }
5420
5421         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5422
5423         my $migrate_port = 0;
5424         my $migrate_uri;
5425         if ($statefile) {
5426             if ($statefile eq 'tcp') {
5427                 my $localip = "localhost";
5428                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5429                 my $nodename = PVE::INotify::nodename();
5430
5431                 if (!defined($migration_type)) {
5432                     if (defined($datacenterconf->{migration}->{type})) {
5433                         $migration_type = $datacenterconf->{migration}->{type};
5434                     } else {
5435                         $migration_type = 'secure';
5436                     }
5437                 }
5438
5439                 if ($migration_type eq 'insecure') {
5440                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5441                     if ($migrate_network_addr) {
5442                         $localip = $migrate_network_addr;
5443                     } else {
5444                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5445                     }
5446
5447                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5448                 }
5449
5450                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5451                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5452                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5453                 push @$cmd, '-incoming', $migrate_uri;
5454                 push @$cmd, '-S';
5455
5456             } elsif ($statefile eq 'unix') {
5457                 # should be default for secure migrations as a ssh TCP forward
5458                 # tunnel is not deterministic reliable ready and fails regurarly
5459                 # to set up in time, so use UNIX socket forwards
5460                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5461                 unlink $socket_addr;
5462
5463                 $migrate_uri = "unix:$socket_addr";
5464
5465                 push @$cmd, '-incoming', $migrate_uri;
5466                 push @$cmd, '-S';
5467
5468             } else {
5469                 push @$cmd, '-loadstate', $statefile;
5470             }
5471         } elsif ($paused) {
5472             push @$cmd, '-S';
5473         }
5474
5475         # host pci devices
5476         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5477           my $d = parse_hostpci($conf->{"hostpci$i"});
5478           next if !$d;
5479           my $pcidevices = $d->{pciid};
5480           foreach my $pcidevice (@$pcidevices) {
5481                 my $pciid = $pcidevice->{id};
5482
5483                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5484                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5485                 die "no pci device info for device '$pciid'\n" if !$info;
5486
5487                 if ($d->{mdev}) {
5488                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5489                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5490                 } else {
5491                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5492                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5493                     die "can't reset pci device '$pciid'\n"
5494                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5495                 }
5496           }
5497         }
5498
5499         PVE::Storage::activate_volumes($storecfg, $vollist);
5500
5501         eval {
5502             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5503                 outfunc => sub {}, errfunc => sub {});
5504         };
5505         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5506         # timeout should be more than enough here...
5507         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5508
5509         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5510                                                   : $defaults->{cpuunits};
5511
5512         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5513         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5514
5515         my %properties = (
5516             Slice => 'qemu.slice',
5517             KillMode => 'none',
5518             CPUShares => $cpuunits
5519         );
5520
5521         if (my $cpulimit = $conf->{cpulimit}) {
5522             $properties{CPUQuota} = int($cpulimit * 100);
5523         }
5524         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5525
5526         my $run_qemu = sub {
5527             PVE::Tools::run_fork sub {
5528                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5529                 run_command($cmd, %run_params);
5530             };
5531         };
5532
5533         if ($conf->{hugepages}) {
5534
5535             my $code = sub {
5536                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5537                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5538
5539                 PVE::QemuServer::Memory::hugepages_mount();
5540                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5541
5542                 eval { $run_qemu->() };
5543                 if (my $err = $@) {
5544                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5545                     die $err;
5546                 }
5547
5548                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5549             };
5550             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5551
5552         } else {
5553             eval { $run_qemu->() };
5554         }
5555
5556         if (my $err = $@) {
5557             # deactivate volumes if start fails
5558             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5559             die "start failed: $err";
5560         }
5561
5562         print "migration listens on $migrate_uri\n" if $migrate_uri;
5563
5564         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5565             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5566             warn $@ if $@;
5567         }
5568
5569         #start nbd server for storage migration
5570         if ($targetstorage) {
5571             my $nodename = PVE::INotify::nodename();
5572             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5573             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5574             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5575             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5576
5577             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5578
5579             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5580
5581             foreach my $opt (sort keys %$local_volumes) {
5582                 my $volid = $local_volumes->{$opt};
5583                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5584                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5585                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5586             }
5587         }
5588
5589         if ($migratedfrom) {
5590             eval {
5591                 set_migration_caps($vmid);
5592             };
5593             warn $@ if $@;
5594
5595             if ($spice_port) {
5596                 print "spice listens on port $spice_port\n";
5597                 if ($spice_ticket) {
5598                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5599                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5600                 }
5601             }
5602
5603         } else {
5604             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5605                 if !$statefile && $conf->{balloon};
5606
5607             foreach my $opt (keys %$conf) {
5608                 next if $opt !~  m/^net\d+$/;
5609                 my $nicconf = parse_net($conf->{$opt});
5610                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5611             }
5612         }
5613
5614         vm_mon_cmd_nocheck($vmid, 'qom-set',
5615                     path => "machine/peripheral/balloon0",
5616                     property => "guest-stats-polling-interval",
5617                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5618
5619         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5620             print "Resumed VM, removing state\n";
5621             delete $conf->@{qw(lock vmstate runningmachine)};
5622             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5623             PVE::Storage::vdisk_free($storecfg, $vmstate);
5624             PVE::QemuConfig->write_config($vmid, $conf);
5625         }
5626
5627         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5628     });
5629 }
5630
5631 sub vm_mon_cmd {
5632     my ($vmid, $execute, %params) = @_;
5633
5634     my $cmd = { execute => $execute, arguments => \%params };
5635     vm_qmp_command($vmid, $cmd);
5636 }
5637
5638 sub vm_mon_cmd_nocheck {
5639     my ($vmid, $execute, %params) = @_;
5640
5641     my $cmd = { execute => $execute, arguments => \%params };
5642     vm_qmp_command($vmid, $cmd, 1);
5643 }
5644
5645 sub vm_qmp_command {
5646     my ($vmid, $cmd, $nocheck) = @_;
5647
5648     my $res;
5649
5650     my $timeout;
5651     if ($cmd->{arguments}) {
5652         $timeout = delete $cmd->{arguments}->{timeout};
5653     }
5654
5655     eval {
5656         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5657         my $sname = qmp_socket($vmid);
5658         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5659             my $qmpclient = PVE::QMPClient->new();
5660
5661             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5662         } else {
5663             die "unable to open monitor socket\n";
5664         }
5665     };
5666     if (my $err = $@) {
5667         syslog("err", "VM $vmid qmp command failed - $err");
5668         die $err;
5669     }
5670
5671     return $res;
5672 }
5673
5674 sub vm_human_monitor_command {
5675     my ($vmid, $cmdline) = @_;
5676
5677     my $cmd = {
5678         execute => 'human-monitor-command',
5679         arguments => { 'command-line' => $cmdline},
5680     };
5681
5682     return vm_qmp_command($vmid, $cmd);
5683 }
5684
5685 sub vm_commandline {
5686     my ($storecfg, $vmid, $snapname) = @_;
5687
5688     my $conf = PVE::QemuConfig->load_config($vmid);
5689
5690     if ($snapname) {
5691         my $snapshot = $conf->{snapshots}->{$snapname};
5692         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5693
5694         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5695
5696         $conf = $snapshot;
5697     }
5698
5699     my $defaults = load_defaults();
5700
5701     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5702
5703     return PVE::Tools::cmd2string($cmd);
5704 }
5705
5706 sub vm_reset {
5707     my ($vmid, $skiplock) = @_;
5708
5709     PVE::QemuConfig->lock_config($vmid, sub {
5710
5711         my $conf = PVE::QemuConfig->load_config($vmid);
5712
5713         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5714
5715         vm_mon_cmd($vmid, "system_reset");
5716     });
5717 }
5718
5719 sub get_vm_volumes {
5720     my ($conf) = @_;
5721
5722     my $vollist = [];
5723     foreach_volid($conf, sub {
5724         my ($volid, $attr) = @_;
5725
5726         return if $volid =~ m|^/|;
5727
5728         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5729         return if !$sid;
5730
5731         push @$vollist, $volid;
5732     });
5733
5734     return $vollist;
5735 }
5736
5737 sub vm_stop_cleanup {
5738     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5739
5740     eval {
5741
5742         if (!$keepActive) {
5743             my $vollist = get_vm_volumes($conf);
5744             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5745         }
5746
5747         foreach my $ext (qw(mon qmp pid vnc qga)) {
5748             unlink "/var/run/qemu-server/${vmid}.$ext";
5749         }
5750
5751         if ($conf->{ivshmem}) {
5752             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5753             # just delete it for now, VMs which have this already open do not
5754             # are affected, but new VMs will get a separated one. If this
5755             # becomes an issue we either add some sort of ref-counting or just
5756             # add a "don't delete on stop" flag to the ivshmem format.
5757             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5758         }
5759
5760         foreach my $key (keys %$conf) {
5761             next if $key !~ m/^hostpci(\d+)$/;
5762             my $hostpciindex = $1;
5763             my $d = parse_hostpci($conf->{$key});
5764             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5765
5766             foreach my $pci (@{$d->{pciid}}) {
5767                 my $pciid = $pci->{id};
5768                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5769             }
5770         }
5771
5772         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5773     };
5774     warn $@ if $@; # avoid errors - just warn
5775 }
5776
5777 # call only in locked context
5778 sub _do_vm_stop {
5779     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5780
5781     my $pid = check_running($vmid, $nocheck);
5782     return if !$pid;
5783
5784     my $conf;
5785     if (!$nocheck) {
5786         $conf = PVE::QemuConfig->load_config($vmid);
5787         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5788         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5789             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5790             $timeout = $opts->{down} if $opts->{down};
5791         }
5792         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5793     }
5794
5795     eval {
5796         if ($shutdown) {
5797             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5798                 vm_qmp_command($vmid, {
5799                         execute => "guest-shutdown",
5800                         arguments => { timeout => $timeout }
5801                     }, $nocheck);
5802             } else {
5803                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5804             }
5805         } else {
5806             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5807         }
5808     };
5809     my $err = $@;
5810
5811     if (!$err) {
5812         $timeout = 60 if !defined($timeout);
5813
5814         my $count = 0;
5815         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5816             $count++;
5817             sleep 1;
5818         }
5819
5820         if ($count >= $timeout) {
5821             if ($force) {
5822                 warn "VM still running - terminating now with SIGTERM\n";
5823                 kill 15, $pid;
5824             } else {
5825                 die "VM quit/powerdown failed - got timeout\n";
5826             }
5827         } else {
5828             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5829             return;
5830         }
5831     } else {
5832         if ($force) {
5833             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5834             kill 15, $pid;
5835         } else {
5836             die "VM quit/powerdown failed\n";
5837         }
5838     }
5839
5840     # wait again
5841     $timeout = 10;
5842
5843     my $count = 0;
5844     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5845         $count++;
5846         sleep 1;
5847     }
5848
5849     if ($count >= $timeout) {
5850         warn "VM still running - terminating now with SIGKILL\n";
5851         kill 9, $pid;
5852         sleep 1;
5853     }
5854
5855     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5856 }
5857
5858 # Note: use $nocheck to skip tests if VM configuration file exists.
5859 # We need that when migration VMs to other nodes (files already moved)
5860 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5861 sub vm_stop {
5862     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5863
5864     $force = 1 if !defined($force) && !$shutdown;
5865
5866     if ($migratedfrom){
5867         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5868         kill 15, $pid if $pid;
5869         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5870         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5871         return;
5872     }
5873
5874     PVE::QemuConfig->lock_config($vmid, sub {
5875         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5876    });
5877 }
5878
5879 sub vm_reboot {
5880     my ($vmid, $timeout) = @_;
5881
5882     PVE::QemuConfig->lock_config($vmid, sub {
5883
5884         # only reboot if running, as qmeventd starts it again on a stop event
5885         return if !check_running($vmid);
5886
5887         create_reboot_request($vmid);
5888
5889         my $storecfg = PVE::Storage::config();
5890         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5891
5892    });
5893 }
5894
5895 sub vm_suspend {
5896     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5897
5898     my $conf;
5899     my $path;
5900     my $storecfg;
5901     my $vmstate;
5902
5903     PVE::QemuConfig->lock_config($vmid, sub {
5904
5905         $conf = PVE::QemuConfig->load_config($vmid);
5906
5907         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5908         PVE::QemuConfig->check_lock($conf)
5909             if !($skiplock || $is_backing_up);
5910
5911         die "cannot suspend to disk during backup\n"
5912             if $is_backing_up && $includestate;
5913
5914         if ($includestate) {
5915             $conf->{lock} = 'suspending';
5916             my $date = strftime("%Y-%m-%d", localtime(time()));
5917             $storecfg = PVE::Storage::config();
5918             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5919             $path = PVE::Storage::path($storecfg, $vmstate);
5920             PVE::QemuConfig->write_config($vmid, $conf);
5921         } else {
5922             vm_mon_cmd($vmid, "stop");
5923         }
5924     });
5925
5926     if ($includestate) {
5927         # save vm state
5928         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5929
5930         eval {
5931             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5932             for(;;) {
5933                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5934                 if (!$state->{status}) {
5935                     die "savevm not active\n";
5936                 } elsif ($state->{status} eq 'active') {
5937                     sleep(1);
5938                     next;
5939                 } elsif ($state->{status} eq 'completed') {
5940                     print "State saved, quitting\n";
5941                     last;
5942                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5943                     die "query-savevm failed with error '$state->{error}'\n"
5944                 } else {
5945                     die "query-savevm returned status '$state->{status}'\n";
5946                 }
5947             }
5948         };
5949         my $err = $@;
5950
5951         PVE::QemuConfig->lock_config($vmid, sub {
5952             $conf = PVE::QemuConfig->load_config($vmid);
5953             if ($err) {
5954                 # cleanup, but leave suspending lock, to indicate something went wrong
5955                 eval {
5956                     vm_mon_cmd($vmid, "savevm-end");
5957                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5958                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5959                     delete $conf->@{qw(vmstate runningmachine)};
5960                     PVE::QemuConfig->write_config($vmid, $conf);
5961                 };
5962                 warn $@ if $@;
5963                 die $err;
5964             }
5965
5966             die "lock changed unexpectedly\n"
5967                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5968
5969             vm_qmp_command($vmid, { execute => "quit" });
5970             $conf->{lock} = 'suspended';
5971             PVE::QemuConfig->write_config($vmid, $conf);
5972         });
5973     }
5974 }
5975
5976 sub vm_resume {
5977     my ($vmid, $skiplock, $nocheck) = @_;
5978
5979     PVE::QemuConfig->lock_config($vmid, sub {
5980         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5981         my $res = $vm_mon_cmd->($vmid, 'query-status');
5982         my $resume_cmd = 'cont';
5983
5984         if ($res->{status} && $res->{status} eq 'suspended') {
5985             $resume_cmd = 'system_wakeup';
5986         }
5987
5988         if (!$nocheck) {
5989
5990             my $conf = PVE::QemuConfig->load_config($vmid);
5991
5992             PVE::QemuConfig->check_lock($conf)
5993                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5994         }
5995
5996         $vm_mon_cmd->($vmid, $resume_cmd);
5997     });
5998 }
5999
6000 sub vm_sendkey {
6001     my ($vmid, $skiplock, $key) = @_;
6002
6003     PVE::QemuConfig->lock_config($vmid, sub {
6004
6005         my $conf = PVE::QemuConfig->load_config($vmid);
6006
6007         # there is no qmp command, so we use the human monitor command
6008         my $res = vm_human_monitor_command($vmid, "sendkey $key");
6009         die $res if $res ne '';
6010     });
6011 }
6012
6013 sub vm_destroy {
6014     my ($storecfg, $vmid, $skiplock) = @_;
6015
6016     PVE::QemuConfig->lock_config($vmid, sub {
6017
6018         my $conf = PVE::QemuConfig->load_config($vmid);
6019
6020         if (!check_running($vmid)) {
6021             destroy_vm($storecfg, $vmid, undef, $skiplock);
6022         } else {
6023             die "VM $vmid is running - destroy failed\n";
6024         }
6025     });
6026 }
6027
6028 # vzdump restore implementaion
6029
6030 sub tar_archive_read_firstfile {
6031     my $archive = shift;
6032
6033     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6034
6035     # try to detect archive type first
6036     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6037         die "unable to open file '$archive'\n";
6038     my $firstfile = <$fh>;
6039     kill 15, $pid;
6040     close $fh;
6041
6042     die "ERROR: archive contaions no data\n" if !$firstfile;
6043     chomp $firstfile;
6044
6045     return $firstfile;
6046 }
6047
6048 sub tar_restore_cleanup {
6049     my ($storecfg, $statfile) = @_;
6050
6051     print STDERR "starting cleanup\n";
6052
6053     if (my $fd = IO::File->new($statfile, "r")) {
6054         while (defined(my $line = <$fd>)) {
6055             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6056                 my $volid = $2;
6057                 eval {
6058                     if ($volid =~ m|^/|) {
6059                         unlink $volid || die 'unlink failed\n';
6060                     } else {
6061                         PVE::Storage::vdisk_free($storecfg, $volid);
6062                     }
6063                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6064                 };
6065                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6066             } else {
6067                 print STDERR "unable to parse line in statfile - $line";
6068             }
6069         }
6070         $fd->close();
6071     }
6072 }
6073
6074 sub restore_archive {
6075     my ($archive, $vmid, $user, $opts) = @_;
6076
6077     my $format = $opts->{format};
6078     my $comp;
6079
6080     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6081         $format = 'tar' if !$format;
6082         $comp = 'gzip';
6083     } elsif ($archive =~ m/\.tar$/) {
6084         $format = 'tar' if !$format;
6085     } elsif ($archive =~ m/.tar.lzo$/) {
6086         $format = 'tar' if !$format;
6087         $comp = 'lzop';
6088     } elsif ($archive =~ m/\.vma$/) {
6089         $format = 'vma' if !$format;
6090     } elsif ($archive =~ m/\.vma\.gz$/) {
6091         $format = 'vma' if !$format;
6092         $comp = 'gzip';
6093     } elsif ($archive =~ m/\.vma\.lzo$/) {
6094         $format = 'vma' if !$format;
6095         $comp = 'lzop';
6096     } else {
6097         $format = 'vma' if !$format; # default
6098     }
6099
6100     # try to detect archive format
6101     if ($format eq 'tar') {
6102         return restore_tar_archive($archive, $vmid, $user, $opts);
6103     } else {
6104         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6105     }
6106 }
6107
6108 sub restore_update_config_line {
6109     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6110
6111     return if $line =~ m/^\#qmdump\#/;
6112     return if $line =~ m/^\#vzdump\#/;
6113     return if $line =~ m/^lock:/;
6114     return if $line =~ m/^unused\d+:/;
6115     return if $line =~ m/^parent:/;
6116
6117     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6118     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6119         # try to convert old 1.X settings
6120         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6121         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6122             my ($model, $macaddr) = split(/\=/, $devconfig);
6123             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6124             my $net = {
6125                 model => $model,
6126                 bridge => "vmbr$ind",
6127                 macaddr => $macaddr,
6128             };
6129             my $netstr = print_net($net);
6130
6131             print $outfd "net$cookie->{netcount}: $netstr\n";
6132             $cookie->{netcount}++;
6133         }
6134     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6135         my ($id, $netstr) = ($1, $2);
6136         my $net = parse_net($netstr);
6137         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6138         $netstr = print_net($net);
6139         print $outfd "$id: $netstr\n";
6140     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6141         my $virtdev = $1;
6142         my $value = $3;
6143         my $di = parse_drive($virtdev, $value);
6144         if (defined($di->{backup}) && !$di->{backup}) {
6145             print $outfd "#$line";
6146         } elsif ($map->{$virtdev}) {
6147             delete $di->{format}; # format can change on restore
6148             $di->{file} = $map->{$virtdev};
6149             $value = print_drive($vmid, $di);
6150             print $outfd "$virtdev: $value\n";
6151         } else {
6152             print $outfd $line;
6153         }
6154     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6155         my $vmgenid = $1;
6156         if ($vmgenid ne '0') {
6157             # always generate a new vmgenid if there was a valid one setup
6158             $vmgenid = generate_uuid();
6159         }
6160         print $outfd "vmgenid: $vmgenid\n";
6161     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6162         my ($uuid, $uuid_str);
6163         UUID::generate($uuid);
6164         UUID::unparse($uuid, $uuid_str);
6165         my $smbios1 = parse_smbios1($2);
6166         $smbios1->{uuid} = $uuid_str;
6167         print $outfd $1.print_smbios1($smbios1)."\n";
6168     } else {
6169         print $outfd $line;
6170     }
6171 }
6172
6173 sub scan_volids {
6174     my ($cfg, $vmid) = @_;
6175
6176     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6177
6178     my $volid_hash = {};
6179     foreach my $storeid (keys %$info) {
6180         foreach my $item (@{$info->{$storeid}}) {
6181             next if !($item->{volid} && $item->{size});
6182             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6183             $volid_hash->{$item->{volid}} = $item;
6184         }
6185     }
6186
6187     return $volid_hash;
6188 }
6189
6190 sub is_volume_in_use {
6191     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6192
6193     my $path = PVE::Storage::path($storecfg, $volid);
6194
6195     my $scan_config = sub {
6196         my ($cref, $snapname) = @_;
6197
6198         foreach my $key (keys %$cref) {
6199             my $value = $cref->{$key};
6200             if (is_valid_drivename($key)) {
6201                 next if $skip_drive && $key eq $skip_drive;
6202                 my $drive = parse_drive($key, $value);
6203                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6204                 return 1 if $volid eq $drive->{file};
6205                 if ($drive->{file} =~ m!^/!) {
6206                     return 1 if $drive->{file} eq $path;
6207                 } else {
6208                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6209                     next if !$storeid;
6210                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6211                     next if !$scfg;
6212                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6213                 }
6214             }
6215         }
6216
6217         return 0;
6218     };
6219
6220     return 1 if &$scan_config($conf);
6221
6222     undef $skip_drive;
6223
6224     foreach my $snapname (keys %{$conf->{snapshots}}) {
6225         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6226     }
6227
6228     return 0;
6229 }
6230
6231 sub update_disksize {
6232     my ($vmid, $conf, $volid_hash) = @_;
6233
6234     my $changes;
6235     my $prefix = "VM $vmid:";
6236
6237     # used and unused disks
6238     my $referenced = {};
6239
6240     # Note: it is allowed to define multiple storages with same path (alias), so
6241     # we need to check both 'volid' and real 'path' (two different volid can point
6242     # to the same path).
6243
6244     my $referencedpath = {};
6245
6246     # update size info
6247     foreach my $opt (keys %$conf) {
6248         if (is_valid_drivename($opt)) {
6249             my $drive = parse_drive($opt, $conf->{$opt});
6250             my $volid = $drive->{file};
6251             next if !$volid;
6252
6253             $referenced->{$volid} = 1;
6254             if ($volid_hash->{$volid} &&
6255                 (my $path = $volid_hash->{$volid}->{path})) {
6256                 $referencedpath->{$path} = 1;
6257             }
6258
6259             next if drive_is_cdrom($drive);
6260             next if !$volid_hash->{$volid};
6261
6262             $drive->{size} = $volid_hash->{$volid}->{size};
6263             my $new = print_drive($vmid, $drive);
6264             if ($new ne $conf->{$opt}) {
6265                 $changes = 1;
6266                 $conf->{$opt} = $new;
6267                 print "$prefix update disk '$opt' information.\n";
6268             }
6269         }
6270     }
6271
6272     # remove 'unusedX' entry if volume is used
6273     foreach my $opt (keys %$conf) {
6274         next if $opt !~ m/^unused\d+$/;
6275         my $volid = $conf->{$opt};
6276         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6277         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6278             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6279             $changes = 1;
6280             delete $conf->{$opt};
6281         }
6282
6283         $referenced->{$volid} = 1;
6284         $referencedpath->{$path} = 1 if $path;
6285     }
6286
6287     foreach my $volid (sort keys %$volid_hash) {
6288         next if $volid =~ m/vm-$vmid-state-/;
6289         next if $referenced->{$volid};
6290         my $path = $volid_hash->{$volid}->{path};
6291         next if !$path; # just to be sure
6292         next if $referencedpath->{$path};
6293         $changes = 1;
6294         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6295         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6296         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6297     }
6298
6299     return $changes;
6300 }
6301
6302 sub rescan {
6303     my ($vmid, $nolock, $dryrun) = @_;
6304
6305     my $cfg = PVE::Storage::config();
6306
6307     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6308     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6309     foreach my $stor (keys %{$cfg->{ids}}) {
6310         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6311     }
6312
6313     print "rescan volumes...\n";
6314     my $volid_hash = scan_volids($cfg, $vmid);
6315
6316     my $updatefn =  sub {
6317         my ($vmid) = @_;
6318
6319         my $conf = PVE::QemuConfig->load_config($vmid);
6320
6321         PVE::QemuConfig->check_lock($conf);
6322
6323         my $vm_volids = {};
6324         foreach my $volid (keys %$volid_hash) {
6325             my $info = $volid_hash->{$volid};
6326             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6327         }
6328
6329         my $changes = update_disksize($vmid, $conf, $vm_volids);
6330
6331         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6332     };
6333
6334     if (defined($vmid)) {
6335         if ($nolock) {
6336             &$updatefn($vmid);
6337         } else {
6338             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6339         }
6340     } else {
6341         my $vmlist = config_list();
6342         foreach my $vmid (keys %$vmlist) {
6343             if ($nolock) {
6344                 &$updatefn($vmid);
6345             } else {
6346                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6347             }
6348         }
6349     }
6350 }
6351
6352 sub restore_vma_archive {
6353     my ($archive, $vmid, $user, $opts, $comp) = @_;
6354
6355     my $readfrom = $archive;
6356
6357     my $cfg = PVE::Storage::config();
6358     my $commands = [];
6359     my $bwlimit = $opts->{bwlimit};
6360
6361     my $dbg_cmdstring = '';
6362     my $add_pipe = sub {
6363         my ($cmd) = @_;
6364         push @$commands, $cmd;
6365         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6366         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6367         $readfrom = '-';
6368     };
6369
6370     my $input = undef;
6371     if ($archive eq '-') {
6372         $input = '<&STDIN';
6373     } else {
6374         # If we use a backup from a PVE defined storage we also consider that
6375         # storage's rate limit:
6376         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6377         if (defined($volid)) {
6378             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6379             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6380             if ($readlimit) {
6381                 print STDERR "applying read rate limit: $readlimit\n";
6382                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6383                 $add_pipe->($cstream);
6384             }
6385         }
6386     }
6387
6388     if ($comp) {
6389         my $cmd;
6390         if ($comp eq 'gzip') {
6391             $cmd = ['zcat', $readfrom];
6392         } elsif ($comp eq 'lzop') {
6393             $cmd = ['lzop', '-d', '-c', $readfrom];
6394         } else {
6395             die "unknown compression method '$comp'\n";
6396         }
6397         $add_pipe->($cmd);
6398     }
6399
6400     my $tmpdir = "/var/tmp/vzdumptmp$$";
6401     rmtree $tmpdir;
6402
6403     # disable interrupts (always do cleanups)
6404     local $SIG{INT} =
6405         local $SIG{TERM} =
6406         local $SIG{QUIT} =
6407         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6408
6409     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6410     POSIX::mkfifo($mapfifo, 0600);
6411     my $fifofh;
6412
6413     my $openfifo = sub {
6414         open($fifofh, '>', $mapfifo) || die $!;
6415     };
6416
6417     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6418
6419     my $oldtimeout;
6420     my $timeout = 5;
6421
6422     my $devinfo = {};
6423
6424     my $rpcenv = PVE::RPCEnvironment::get();
6425
6426     my $conffile = PVE::QemuConfig->config_file($vmid);
6427     my $tmpfn = "$conffile.$$.tmp";
6428
6429     # Note: $oldconf is undef if VM does not exists
6430     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6431     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6432
6433     my %storage_limits;
6434
6435     my $print_devmap = sub {
6436         my $virtdev_hash = {};
6437
6438         my $cfgfn = "$tmpdir/qemu-server.conf";
6439
6440         # we can read the config - that is already extracted
6441         my $fh = IO::File->new($cfgfn, "r") ||
6442             "unable to read qemu-server.conf - $!\n";
6443
6444         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6445         if (-f $fwcfgfn) {
6446             my $pve_firewall_dir = '/etc/pve/firewall';
6447             mkdir $pve_firewall_dir; # make sure the dir exists
6448             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6449         }
6450
6451         while (defined(my $line = <$fh>)) {
6452             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6453                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6454                 die "archive does not contain data for drive '$virtdev'\n"
6455                     if !$devinfo->{$devname};
6456                 if (defined($opts->{storage})) {
6457                     $storeid = $opts->{storage} || 'local';
6458                 } elsif (!$storeid) {
6459                     $storeid = 'local';
6460                 }
6461                 $format = 'raw' if !$format;
6462                 $devinfo->{$devname}->{devname} = $devname;
6463                 $devinfo->{$devname}->{virtdev} = $virtdev;
6464                 $devinfo->{$devname}->{format} = $format;
6465                 $devinfo->{$devname}->{storeid} = $storeid;
6466
6467                 # check permission on storage
6468                 my $pool = $opts->{pool}; # todo: do we need that?
6469                 if ($user ne 'root@pam') {
6470                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6471                 }
6472
6473                 $storage_limits{$storeid} = $bwlimit;
6474
6475                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6476             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6477                 my $virtdev = $1;
6478                 my $drive = parse_drive($virtdev, $2);
6479                 if (drive_is_cloudinit($drive)) {
6480                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6481                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6482                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6483
6484                     my $d = {
6485                         format => $format,
6486                         storeid => $opts->{storage} // $storeid,
6487                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6488                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6489                         name => "vm-$vmid-cloudinit",
6490                         is_cloudinit => 1,
6491                     };
6492                     $virtdev_hash->{$virtdev} = $d;
6493                 }
6494             }
6495         }
6496
6497         foreach my $key (keys %storage_limits) {
6498             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6499             next if !$limit;
6500             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6501             $storage_limits{$key} = $limit * 1024;
6502         }
6503
6504         foreach my $devname (keys %$devinfo) {
6505             die "found no device mapping information for device '$devname'\n"
6506                 if !$devinfo->{$devname}->{virtdev};
6507         }
6508
6509         # create empty/temp config
6510         if ($oldconf) {
6511             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6512             foreach_drive($oldconf, sub {
6513                 my ($ds, $drive) = @_;
6514
6515                 return if drive_is_cdrom($drive, 1);
6516
6517                 my $volid = $drive->{file};
6518                 return if !$volid || $volid =~ m|^/|;
6519
6520                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6521                 return if !$path || !$owner || ($owner != $vmid);
6522
6523                 # Note: only delete disk we want to restore
6524                 # other volumes will become unused
6525                 if ($virtdev_hash->{$ds}) {
6526                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6527                     if (my $err = $@) {
6528                         warn $err;
6529                     }
6530                 }
6531             });
6532
6533             # delete vmstate files, after the restore we have no snapshots anymore
6534             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6535                 my $snap = $oldconf->{snapshots}->{$snapname};
6536                 if ($snap->{vmstate}) {
6537                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6538                     if (my $err = $@) {
6539                         warn $err;
6540                     }
6541                 }
6542             }
6543         }
6544
6545         my $map = {};
6546         foreach my $virtdev (sort keys %$virtdev_hash) {
6547             my $d = $virtdev_hash->{$virtdev};
6548             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6549             my $storeid = $d->{storeid};
6550             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6551
6552             my $map_opts = '';
6553             if (my $limit = $storage_limits{$storeid}) {
6554                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6555             }
6556
6557             # test if requested format is supported
6558             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6559             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6560             $d->{format} = $defFormat if !$supported;
6561
6562             my $name;
6563             if ($d->{is_cloudinit}) {
6564                 $name = $d->{name};
6565                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6566             }
6567
6568             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6569             print STDERR "new volume ID is '$volid'\n";
6570             $d->{volid} = $volid;
6571
6572             PVE::Storage::activate_volumes($cfg, [$volid]);
6573
6574             my $write_zeros = 1;
6575             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6576                 $write_zeros = 0;
6577             }
6578
6579             if (!$d->{is_cloudinit}) {
6580                 my $path = PVE::Storage::path($cfg, $volid);
6581
6582                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6583
6584                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6585             }
6586             $map->{$virtdev} = $volid;
6587         }
6588
6589         $fh->seek(0, 0) || die "seek failed - $!\n";
6590
6591         my $outfd = new IO::File ($tmpfn, "w") ||
6592             die "unable to write config for VM $vmid\n";
6593
6594         my $cookie = { netcount => 0 };
6595         while (defined(my $line = <$fh>)) {
6596             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6597         }
6598
6599         $fh->close();
6600         $outfd->close();
6601     };
6602
6603     eval {
6604         # enable interrupts
6605         local $SIG{INT} =
6606             local $SIG{TERM} =
6607             local $SIG{QUIT} =
6608             local $SIG{HUP} =
6609             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6610         local $SIG{ALRM} = sub { die "got timeout\n"; };
6611
6612         $oldtimeout = alarm($timeout);
6613
6614         my $parser = sub {
6615             my $line = shift;
6616
6617             print "$line\n";
6618
6619             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6620                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6621                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6622             } elsif ($line =~ m/^CTIME: /) {
6623                 # we correctly received the vma config, so we can disable
6624                 # the timeout now for disk allocation (set to 10 minutes, so
6625                 # that we always timeout if something goes wrong)
6626                 alarm(600);
6627                 &$print_devmap();
6628                 print $fifofh "done\n";
6629                 my $tmp = $oldtimeout || 0;
6630                 $oldtimeout = undef;
6631                 alarm($tmp);
6632                 close($fifofh);
6633             }
6634         };
6635
6636         print "restore vma archive: $dbg_cmdstring\n";
6637         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6638     };
6639     my $err = $@;
6640
6641     alarm($oldtimeout) if $oldtimeout;
6642
6643     my $vollist = [];
6644     foreach my $devname (keys %$devinfo) {
6645         my $volid = $devinfo->{$devname}->{volid};
6646         push @$vollist, $volid if $volid;
6647     }
6648
6649     PVE::Storage::deactivate_volumes($cfg, $vollist);
6650
6651     unlink $mapfifo;
6652
6653     if ($err) {
6654         rmtree $tmpdir;
6655         unlink $tmpfn;
6656
6657         foreach my $devname (keys %$devinfo) {
6658             my $volid = $devinfo->{$devname}->{volid};
6659             next if !$volid;
6660             eval {
6661                 if ($volid =~ m|^/|) {
6662                     unlink $volid || die 'unlink failed\n';
6663                 } else {
6664                     PVE::Storage::vdisk_free($cfg, $volid);
6665                 }
6666                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6667             };
6668             print STDERR "unable to cleanup '$volid' - $@" if $@;
6669         }
6670         die $err;
6671     }
6672
6673     rmtree $tmpdir;
6674
6675     rename($tmpfn, $conffile) ||
6676         die "unable to commit configuration file '$conffile'\n";
6677
6678     PVE::Cluster::cfs_update(); # make sure we read new file
6679
6680     eval { rescan($vmid, 1); };
6681     warn $@ if $@;
6682 }
6683
6684 sub restore_tar_archive {
6685     my ($archive, $vmid, $user, $opts) = @_;
6686
6687     if ($archive ne '-') {
6688         my $firstfile = tar_archive_read_firstfile($archive);
6689         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6690             if $firstfile ne 'qemu-server.conf';
6691     }
6692
6693     my $storecfg = PVE::Storage::config();
6694
6695     # destroy existing data - keep empty config
6696     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6697     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6698
6699     my $tocmd = "/usr/lib/qemu-server/qmextract";
6700
6701     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6702     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6703     $tocmd .= ' --prealloc' if $opts->{prealloc};
6704     $tocmd .= ' --info' if $opts->{info};
6705
6706     # tar option "xf" does not autodetect compression when read from STDIN,
6707     # so we pipe to zcat
6708     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6709         PVE::Tools::shellquote("--to-command=$tocmd");
6710
6711     my $tmpdir = "/var/tmp/vzdumptmp$$";
6712     mkpath $tmpdir;
6713
6714     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6715     local $ENV{VZDUMP_VMID} = $vmid;
6716     local $ENV{VZDUMP_USER} = $user;
6717
6718     my $conffile = PVE::QemuConfig->config_file($vmid);
6719     my $tmpfn = "$conffile.$$.tmp";
6720
6721     # disable interrupts (always do cleanups)
6722     local $SIG{INT} =
6723         local $SIG{TERM} =
6724         local $SIG{QUIT} =
6725         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6726
6727     eval {
6728         # enable interrupts
6729         local $SIG{INT} =
6730             local $SIG{TERM} =
6731             local $SIG{QUIT} =
6732             local $SIG{HUP} =
6733             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6734
6735         if ($archive eq '-') {
6736             print "extracting archive from STDIN\n";
6737             run_command($cmd, input => "<&STDIN");
6738         } else {
6739             print "extracting archive '$archive'\n";
6740             run_command($cmd);
6741         }
6742
6743         return if $opts->{info};
6744
6745         # read new mapping
6746         my $map = {};
6747         my $statfile = "$tmpdir/qmrestore.stat";
6748         if (my $fd = IO::File->new($statfile, "r")) {
6749             while (defined (my $line = <$fd>)) {
6750                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6751                     $map->{$1} = $2 if $1;
6752                 } else {
6753                     print STDERR "unable to parse line in statfile - $line\n";
6754                 }
6755             }
6756             $fd->close();
6757         }
6758
6759         my $confsrc = "$tmpdir/qemu-server.conf";
6760
6761         my $srcfd = new IO::File($confsrc, "r") ||
6762             die "unable to open file '$confsrc'\n";
6763
6764         my $outfd = new IO::File ($tmpfn, "w") ||
6765             die "unable to write config for VM $vmid\n";
6766
6767         my $cookie = { netcount => 0 };
6768         while (defined (my $line = <$srcfd>)) {
6769             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6770         }
6771
6772         $srcfd->close();
6773         $outfd->close();
6774     };
6775     my $err = $@;
6776
6777     if ($err) {
6778
6779         unlink $tmpfn;
6780
6781         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6782
6783         die $err;
6784     }
6785
6786     rmtree $tmpdir;
6787
6788     rename $tmpfn, $conffile ||
6789         die "unable to commit configuration file '$conffile'\n";
6790
6791     PVE::Cluster::cfs_update(); # make sure we read new file
6792
6793     eval { rescan($vmid, 1); };
6794     warn $@ if $@;
6795 };
6796
6797 sub foreach_storage_used_by_vm {
6798     my ($conf, $func) = @_;
6799
6800     my $sidhash = {};
6801
6802     foreach_drive($conf, sub {
6803         my ($ds, $drive) = @_;
6804         return if drive_is_cdrom($drive);
6805
6806         my $volid = $drive->{file};
6807
6808         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6809         $sidhash->{$sid} = $sid if $sid;
6810     });
6811
6812     foreach my $sid (sort keys %$sidhash) {
6813         &$func($sid);
6814     }
6815 }
6816
6817 sub do_snapshots_with_qemu {
6818     my ($storecfg, $volid) = @_;
6819
6820     my $storage_name = PVE::Storage::parse_volume_id($volid);
6821     my $scfg = $storecfg->{ids}->{$storage_name};
6822
6823     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6824         return 1;
6825     }
6826
6827     if ($volid =~ m/\.(qcow2|qed)$/){
6828         return 1;
6829     }
6830
6831     return undef;
6832 }
6833
6834 sub qga_check_running {
6835     my ($vmid, $nowarn) = @_;
6836
6837     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6838     if ($@) {
6839         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6840         return 0;
6841     }
6842     return 1;
6843 }
6844
6845 sub template_create {
6846     my ($vmid, $conf, $disk) = @_;
6847
6848     my $storecfg = PVE::Storage::config();
6849
6850     foreach_drive($conf, sub {
6851         my ($ds, $drive) = @_;
6852
6853         return if drive_is_cdrom($drive);
6854         return if $disk && $ds ne $disk;
6855
6856         my $volid = $drive->{file};
6857         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6858
6859         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6860         $drive->{file} = $voliddst;
6861         $conf->{$ds} = print_drive($vmid, $drive);
6862         PVE::QemuConfig->write_config($vmid, $conf);
6863     });
6864 }
6865
6866 sub convert_iscsi_path {
6867     my ($path) = @_;
6868
6869     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6870         my $portal = $1;
6871         my $target = $2;
6872         my $lun = $3;
6873
6874         my $initiator_name = get_initiator_name();
6875
6876         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6877                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6878     }
6879
6880     die "cannot convert iscsi path '$path', unkown format\n";
6881 }
6882
6883 sub qemu_img_convert {
6884     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6885
6886     my $storecfg = PVE::Storage::config();
6887     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6888     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6889
6890     if ($src_storeid && $dst_storeid) {
6891
6892         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6893
6894         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6895         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6896
6897         my $src_format = qemu_img_format($src_scfg, $src_volname);
6898         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6899
6900         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6901         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6902
6903         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6904         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6905
6906         my $cmd = [];
6907         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6908         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6909         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6910         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6911
6912         if ($src_is_iscsi) {
6913             push @$cmd, '--image-opts';
6914             $src_path = convert_iscsi_path($src_path);
6915         } else {
6916             push @$cmd, '-f', $src_format;
6917         }
6918
6919         if ($dst_is_iscsi) {
6920             push @$cmd, '--target-image-opts';
6921             $dst_path = convert_iscsi_path($dst_path);
6922         } else {
6923             push @$cmd, '-O', $dst_format;
6924         }
6925
6926         push @$cmd, $src_path;
6927
6928         if (!$dst_is_iscsi && $is_zero_initialized) {
6929             push @$cmd, "zeroinit:$dst_path";
6930         } else {
6931             push @$cmd, $dst_path;
6932         }
6933
6934         my $parser = sub {
6935             my $line = shift;
6936             if($line =~ m/\((\S+)\/100\%\)/){
6937                 my $percent = $1;
6938                 my $transferred = int($size * $percent / 100);
6939                 my $remaining = $size - $transferred;
6940
6941                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6942             }
6943
6944         };
6945
6946         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6947         my $err = $@;
6948         die "copy failed: $err" if $err;
6949     }
6950 }
6951
6952 sub qemu_img_format {
6953     my ($scfg, $volname) = @_;
6954
6955     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6956         return $1;
6957     } else {
6958         return "raw";
6959     }
6960 }
6961
6962 sub qemu_drive_mirror {
6963     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6964
6965     $jobs = {} if !$jobs;
6966
6967     my $qemu_target;
6968     my $format;
6969     $jobs->{"drive-$drive"} = {};
6970
6971     if ($dst_volid =~ /^nbd:/) {
6972         $qemu_target = $dst_volid;
6973         $format = "nbd";
6974     } else {
6975         my $storecfg = PVE::Storage::config();
6976         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6977
6978         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6979
6980         $format = qemu_img_format($dst_scfg, $dst_volname);
6981
6982         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6983
6984         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6985     }
6986
6987     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6988     $opts->{format} = $format if $format;
6989
6990     if (defined($bwlimit)) {
6991         $opts->{speed} = $bwlimit * 1024;
6992         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6993     } else {
6994         print "drive mirror is starting for drive-$drive\n";
6995     }
6996
6997     # if a job already runs for this device we get an error, catch it for cleanup
6998     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6999     if (my $err = $@) {
7000         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7001         warn "$@\n" if $@;
7002         die "mirroring error: $err\n";
7003     }
7004
7005     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7006 }
7007
7008 sub qemu_drive_mirror_monitor {
7009     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7010
7011     eval {
7012         my $err_complete = 0;
7013
7014         while (1) {
7015             die "storage migration timed out\n" if $err_complete > 300;
7016
7017             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7018
7019             my $running_mirror_jobs = {};
7020             foreach my $stat (@$stats) {
7021                 next if $stat->{type} ne 'mirror';
7022                 $running_mirror_jobs->{$stat->{device}} = $stat;
7023             }
7024
7025             my $readycounter = 0;
7026
7027             foreach my $job (keys %$jobs) {
7028
7029                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7030                     print "$job : finished\n";
7031                     delete $jobs->{$job};
7032                     next;
7033                 }
7034
7035                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7036
7037                 my $busy = $running_mirror_jobs->{$job}->{busy};
7038                 my $ready = $running_mirror_jobs->{$job}->{ready};
7039                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7040                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7041                     my $remaining = $total - $transferred;
7042                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7043
7044                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7045                 }
7046
7047                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7048             }
7049
7050             last if scalar(keys %$jobs) == 0;
7051
7052             if ($readycounter == scalar(keys %$jobs)) {
7053                 print "all mirroring jobs are ready \n";
7054                 last if $skipcomplete; #do the complete later
7055
7056                 if ($vmiddst && $vmiddst != $vmid) {
7057                     my $agent_running = $qga && qga_check_running($vmid);
7058                     if ($agent_running) {
7059                         print "freeze filesystem\n";
7060                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7061                     } else {
7062                         print "suspend vm\n";
7063                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7064                     }
7065
7066                     # if we clone a disk for a new target vm, we don't switch the disk
7067                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7068
7069                     if ($agent_running) {
7070                         print "unfreeze filesystem\n";
7071                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7072                     } else {
7073                         print "resume vm\n";
7074                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7075                     }
7076
7077                     last;
7078                 } else {
7079
7080                     foreach my $job (keys %$jobs) {
7081                         # try to switch the disk if source and destination are on the same guest
7082                         print "$job: Completing block job...\n";
7083
7084                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7085                         if ($@ =~ m/cannot be completed/) {
7086                             print "$job: Block job cannot be completed, try again.\n";
7087                             $err_complete++;
7088                         }else {
7089                             print "$job: Completed successfully.\n";
7090                             $jobs->{$job}->{complete} = 1;
7091                         }
7092                     }
7093                 }
7094             }
7095             sleep 1;
7096         }
7097     };
7098     my $err = $@;
7099
7100     if ($err) {
7101         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7102         die "mirroring error: $err";
7103     }
7104
7105 }
7106
7107 sub qemu_blockjobs_cancel {
7108     my ($vmid, $jobs) = @_;
7109
7110     foreach my $job (keys %$jobs) {
7111         print "$job: Cancelling block job\n";
7112         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7113         $jobs->{$job}->{cancel} = 1;
7114     }
7115
7116     while (1) {
7117         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7118
7119         my $running_jobs = {};
7120         foreach my $stat (@$stats) {
7121             $running_jobs->{$stat->{device}} = $stat;
7122         }
7123
7124         foreach my $job (keys %$jobs) {
7125
7126             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7127                 print "$job: Done.\n";
7128                 delete $jobs->{$job};
7129             }
7130         }
7131
7132         last if scalar(keys %$jobs) == 0;
7133
7134         sleep 1;
7135     }
7136 }
7137
7138 sub clone_disk {
7139     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7140         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7141
7142     my $newvolid;
7143
7144     if (!$full) {
7145         print "create linked clone of drive $drivename ($drive->{file})\n";
7146         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7147         push @$newvollist, $newvolid;
7148     } else {
7149
7150         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7151         $storeid = $storage if $storage;
7152
7153         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7154         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7155
7156         print "create full clone of drive $drivename ($drive->{file})\n";
7157         my $name = undef;
7158         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7159         push @$newvollist, $newvolid;
7160
7161         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7162
7163         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7164         if (!$running || $snapname) {
7165             # TODO: handle bwlimits
7166             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7167         } else {
7168
7169             my $kvmver = get_running_qemu_version ($vmid);
7170             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7171                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7172                     if $drive->{iothread};
7173             }
7174
7175             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7176         }
7177     }
7178
7179     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7180
7181     my $disk = $drive;
7182     $disk->{format} = undef;
7183     $disk->{file} = $newvolid;
7184     $disk->{size} = $size;
7185
7186     return $disk;
7187 }
7188
7189 # this only works if VM is running
7190 sub get_current_qemu_machine {
7191     my ($vmid) = @_;
7192
7193     my $cmd = { execute => 'query-machines', arguments => {} };
7194     my $res = vm_qmp_command($vmid, $cmd);
7195
7196     my ($current, $default);
7197     foreach my $e (@$res) {
7198         $default = $e->{name} if $e->{'is-default'};
7199         $current = $e->{name} if $e->{'is-current'};
7200     }
7201
7202     # fallback to the default machine if current is not supported by qemu
7203     return $current || $default || 'pc';
7204 }
7205
7206 sub get_running_qemu_version {
7207     my ($vmid) = @_;
7208     my $cmd = { execute => 'query-version', arguments => {} };
7209     my $res = vm_qmp_command($vmid, $cmd);
7210     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7211 }
7212
7213 sub qemu_machine_feature_enabled {
7214     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7215
7216     my $current_major;
7217     my $current_minor;
7218
7219     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7220
7221         $current_major = $3;
7222         $current_minor = $4;
7223
7224     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7225
7226         $current_major = $1;
7227         $current_minor = $2;
7228     }
7229
7230     return 1 if $current_major > $version_major ||
7231                 ($current_major == $version_major &&
7232                  $current_minor >= $version_minor);
7233 }
7234
7235 sub qemu_machine_pxe {
7236     my ($vmid, $conf) = @_;
7237
7238     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7239
7240     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7241         $machine .= '.pxe';
7242     }
7243
7244     return $machine;
7245 }
7246
7247 sub qemu_use_old_bios_files {
7248     my ($machine_type) = @_;
7249
7250     return if !$machine_type;
7251
7252     my $use_old_bios_files = undef;
7253
7254     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7255         $machine_type = $1;
7256         $use_old_bios_files = 1;
7257     } else {
7258         my $kvmver = kvm_user_version();
7259         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7260         # load new efi bios files on migration. So this hack is required to allow
7261         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7262         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7263         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7264     }
7265
7266     return ($use_old_bios_files, $machine_type);
7267 }
7268
7269 sub create_efidisk($$$$$) {
7270     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7271
7272     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7273     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7274
7275     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7276     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7277     PVE::Storage::activate_volumes($storecfg, [$volid]);
7278
7279     my $path = PVE::Storage::path($storecfg, $volid);
7280     eval {
7281         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7282     };
7283     die "Copying EFI vars image failed: $@" if $@;
7284
7285     return ($volid, $vars_size);
7286 }
7287
7288 sub vm_iothreads_list {
7289     my ($vmid) = @_;
7290
7291     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7292
7293     my $iothreads = {};
7294     foreach my $iothread (@$res) {
7295         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7296     }
7297
7298     return $iothreads;
7299 }
7300
7301 sub scsihw_infos {
7302     my ($conf, $drive) = @_;
7303
7304     my $maxdev = 0;
7305
7306     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7307         $maxdev = 7;
7308     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7309         $maxdev = 1;
7310     } else {
7311         $maxdev = 256;
7312     }
7313
7314     my $controller = int($drive->{index} / $maxdev);
7315     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7316
7317     return ($maxdev, $controller, $controller_prefix);
7318 }
7319
7320 sub add_hyperv_enlightenments {
7321     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7322
7323     return if $winversion < 6;
7324     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7325
7326     if ($gpu_passthrough || defined($hv_vendor_id)) {
7327         $hv_vendor_id //= 'proxmox';
7328         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7329     }
7330
7331     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7332         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7333         push @$cpuFlags , 'hv_vapic';
7334         push @$cpuFlags , 'hv_time';
7335     } else {
7336         push @$cpuFlags , 'hv_spinlocks=0xffff';
7337     }
7338
7339     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7340         push @$cpuFlags , 'hv_reset';
7341         push @$cpuFlags , 'hv_vpindex';
7342         push @$cpuFlags , 'hv_runtime';
7343     }
7344
7345     if ($winversion >= 7) {
7346         push @$cpuFlags , 'hv_relaxed';
7347
7348         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7349             push @$cpuFlags , 'hv_synic';
7350             push @$cpuFlags , 'hv_stimer';
7351         }
7352
7353         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7354             push @$cpuFlags , 'hv_ipi';
7355         }
7356     }
7357 }
7358
7359 sub windows_version {
7360     my ($ostype) = @_;
7361
7362     return 0 if !$ostype;
7363
7364     my $winversion = 0;
7365
7366     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7367         $winversion = 5;
7368     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7369         $winversion = 6;
7370     } elsif ($ostype =~ m/^win(\d+)$/) {
7371         $winversion = $1;
7372     }
7373
7374     return $winversion;
7375 }
7376
7377 sub resolve_dst_disk_format {
7378         my ($storecfg, $storeid, $src_volname, $format) = @_;
7379         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7380
7381         if (!$format) {
7382             # if no target format is specified, use the source disk format as hint
7383             if ($src_volname) {
7384                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7385                 $format = qemu_img_format($scfg, $src_volname);
7386             } else {
7387                 return $defFormat;
7388             }
7389         }
7390
7391         # test if requested format is supported - else use default
7392         my $supported = grep { $_ eq $format } @$validFormats;
7393         $format = $defFormat if !$supported;
7394         return $format;
7395 }
7396
7397 sub resolve_first_disk {
7398     my $conf = shift;
7399     my @disks = PVE::QemuServer::valid_drive_names();
7400     my $firstdisk;
7401     foreach my $ds (reverse @disks) {
7402         next if !$conf->{$ds};
7403         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7404         next if PVE::QemuServer::drive_is_cdrom($disk);
7405         $firstdisk = $ds;
7406     }
7407     return $firstdisk;
7408 }
7409
7410 sub generate_uuid {
7411     my ($uuid, $uuid_str);
7412     UUID::generate($uuid);
7413     UUID::unparse($uuid, $uuid_str);
7414     return $uuid_str;
7415 }
7416
7417 sub generate_smbios1_uuid {
7418     return "uuid=".generate_uuid();
7419 }
7420
7421 sub nbd_stop {
7422     my ($vmid) = @_;
7423
7424     vm_mon_cmd($vmid, 'nbd-server-stop');
7425 }
7426
7427 sub create_reboot_request {
7428     my ($vmid) = @_;
7429     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7430         or die "failed to create reboot trigger file: $!\n";
7431     close($fh);
7432 }
7433
7434 sub clear_reboot_request {
7435     my ($vmid) = @_;
7436     my $path = "/run/qemu-server/$vmid.reboot";
7437     my $res = 0;
7438
7439     $res = unlink($path);
7440     die "could not remove reboot request for $vmid: $!"
7441         if !$res && $! != POSIX::ENOENT;
7442
7443     return $res;
7444 }
7445
7446 # bash completion helper
7447
7448 sub complete_backup_archives {
7449     my ($cmdname, $pname, $cvalue) = @_;
7450
7451     my $cfg = PVE::Storage::config();
7452
7453     my $storeid;
7454
7455     if ($cvalue =~ m/^([^:]+):/) {
7456         $storeid = $1;
7457     }
7458
7459     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7460
7461     my $res = [];
7462     foreach my $id (keys %$data) {
7463         foreach my $item (@{$data->{$id}}) {
7464             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7465             push @$res, $item->{volid} if defined($item->{volid});
7466         }
7467     }
7468
7469     return $res;
7470 }
7471
7472 my $complete_vmid_full = sub {
7473     my ($running) = @_;
7474
7475     my $idlist = vmstatus();
7476
7477     my $res = [];
7478
7479     foreach my $id (keys %$idlist) {
7480         my $d = $idlist->{$id};
7481         if (defined($running)) {
7482             next if $d->{template};
7483             next if $running && $d->{status} ne 'running';
7484             next if !$running && $d->{status} eq 'running';
7485         }
7486         push @$res, $id;
7487
7488     }
7489     return $res;
7490 };
7491
7492 sub complete_vmid {
7493     return &$complete_vmid_full();
7494 }
7495
7496 sub complete_vmid_stopped {
7497     return &$complete_vmid_full(0);
7498 }
7499
7500 sub complete_vmid_running {
7501     return &$complete_vmid_full(1);
7502 }
7503
7504 sub complete_storage {
7505
7506     my $cfg = PVE::Storage::config();
7507     my $ids = $cfg->{ids};
7508
7509     my $res = [];
7510     foreach my $sid (keys %$ids) {
7511         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7512         next if !$ids->{$sid}->{content}->{images};
7513         push @$res, $sid;
7514     }
7515
7516     return $res;
7517 }
7518
7519 1;