PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my @supported_cpu_flags = (
 171     'pcid',
 172     'spec-ctrl',
 173     'ibpb',
 174     'ssbd',
 175     'virt-ssbd',
 176     'amd-ssbd',
 177     'amd-no-ssb',
 178     'pdpe1gb',
 179     'md-clear',
 180     'hv-tlbflush',
 181     'hv-evmcs',
 182     'aes'
 183 );
 184 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 185
 186 my $cpu_fmt = {
 187     cputype => {
 188         description => "Emulated CPU type.",
 189         type => 'string',
 190         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 191         default => 'kvm64',
 192         default_key => 1,
 193     },
 194     hidden => {
 195         description => "Do not identify as a KVM virtual machine.",
 196         type => 'boolean',
 197         optional => 1,
 198         default => 0
 199     },
 200     'hv-vendor-id' => {
 201         type => 'string',
 202         pattern => qr/[a-zA-Z0-9]{1,12}/,
 203         format_description => 'vendor-id',
 204         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 205         optional => 1,
 206     },
 207     flags => {
 208         description => "List of additional CPU flags separated by ';'."
 209                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 210                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 211         format_description => '+FLAG[;-FLAG...]',
 212         type => 'string',
 213         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 214         optional => 1,
 215     },
 216 };
 217
 218 my $watchdog_fmt = {
 219     model => {
 220         default_key => 1,
 221         type => 'string',
 222         enum => [qw(i6300esb ib700)],
 223         description => "Watchdog type to emulate.",
 224         default => 'i6300esb',
 225         optional => 1,
 226     },
 227     action => {
 228         type => 'string',
 229         enum => [qw(reset shutdown poweroff pause debug none)],
 230         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 231         optional => 1,
 232     },
 233 };
 234 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 235
 236 my $agent_fmt = {
 237     enabled => {
 238         description => "Enable/disable Qemu GuestAgent.",
 239         type => 'boolean',
 240         default => 0,
 241         default_key => 1,
 242     },
 243     fstrim_cloned_disks => {
 244         description => "Run fstrim after cloning/moving a disk.",
 245         type => 'boolean',
 246         optional => 1,
 247         default => 0
 248     },
 249 };
 250
 251 my $vga_fmt = {
 252     type => {
 253         description => "Select the VGA type.",
 254         type => 'string',
 255         default => 'std',
 256         optional => 1,
 257         default_key => 1,
 258         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 259     },
 260     memory => {
 261         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 262         type => 'integer',
 263         optional => 1,
 264         minimum => 4,
 265         maximum => 512,
 266     },
 267 };
 268
 269 my $ivshmem_fmt = {
 270     size => {
 271         type => 'integer',
 272         minimum => 1,
 273         description => "The size of the file in MB.",
 274     },
 275     name => {
 276         type => 'string',
 277         pattern => '[a-zA-Z0-9\-]+',
 278         optional => 1,
 279         format_description => 'string',
 280         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 281     },
 282 };
 283
 284 my $audio_fmt = {
 285     device => {
 286         type => 'string',
 287         enum => [qw(ich9-intel-hda intel-hda AC97)],
 288         description =>  "Configure an audio device."
 289     },
 290     driver =>  {
 291         type => 'string',
 292         enum => ['spice'],
 293         default => 'spice',
 294         optional => 1,
 295         description => "Driver backend for the audio device."
 296     },
 297 };
 298
 299 my $spice_enhancements_fmt = {
 300     foldersharing => {
 301         type => 'boolean',
 302         optional => 1,
 303         default => '0',
 304         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 305     },
 306     videostreaming =>  {
 307         type => 'string',
 308         enum => ['off', 'all', 'filter'],
 309         default => 'off',
 310         optional => 1,
 311         description => "Enable video streaming. Uses compression for detected video streams."
 312     },
 313 };
 314
 315 my $confdesc = {
 316     onboot => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Specifies whether a VM will be started during system bootup.",
 320         default => 0,
 321     },
 322     autostart => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Automatic restart after crash (currently ignored).",
 326         default => 0,
 327     },
 328     hotplug => {
 329         optional => 1,
 330         type => 'string', format => 'pve-hotplug-features',
 331         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 332         default => 'network,disk,usb',
 333     },
 334     reboot => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 338         default => 1,
 339     },
 340     lock => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Lock/unlock the VM.",
 344         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 345     },
 346     cpulimit => {
 347         optional => 1,
 348         type => 'number',
 349         description => "Limit of CPU usage.",
 350         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 351         minimum => 0,
 352         maximum => 128,
 353         default => 0,
 354     },
 355     cpuunits => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "CPU weight for a VM.",
 359         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 360         minimum => 2,
 361         maximum => 262144,
 362         default => 1024,
 363     },
 364     memory => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 368         minimum => 16,
 369         default => 512,
 370     },
 371     balloon => {
 372         optional => 1,
 373         type => 'integer',
 374         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 375         minimum => 0,
 376     },
 377     shares => {
 378         optional => 1,
 379         type => 'integer',
 380         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 381         minimum => 0,
 382         maximum => 50000,
 383         default => 1000,
 384     },
 385     keyboard => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 389                        "It should not be necessary to set it.",
 390         enum => PVE::Tools::kvmkeymaplist(),
 391         default => undef,
 392     },
 393     name => {
 394         optional => 1,
 395         type => 'string', format => 'dns-name',
 396         description => "Set a name for the VM. Only used on the configuration web interface.",
 397     },
 398     scsihw => {
 399         optional => 1,
 400         type => 'string',
 401         description => "SCSI controller model",
 402         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 403         default => 'lsi',
 404     },
 405     description => {
 406         optional => 1,
 407         type => 'string',
 408         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 409     },
 410     ostype => {
 411         optional => 1,
 412         type => 'string',
 413         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 414         description => "Specify guest operating system.",
 415         verbose_description => <<EODESC,
 416 Specify guest operating system. This is used to enable special
 417 optimization/features for specific operating systems:
 418
 419 [horizontal]
 420 other;; unspecified OS
 421 wxp;; Microsoft Windows XP
 422 w2k;; Microsoft Windows 2000
 423 w2k3;; Microsoft Windows 2003
 424 w2k8;; Microsoft Windows 2008
 425 wvista;; Microsoft Windows Vista
 426 win7;; Microsoft Windows 7
 427 win8;; Microsoft Windows 8/2012/2012r2
 428 win10;; Microsoft Windows 10/2016
 429 l24;; Linux 2.4 Kernel
 430 l26;; Linux 2.6/3.X Kernel
 431 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 432 EODESC
 433     },
 434     boot => {
 435         optional => 1,
 436         type => 'string',
 437         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 438         pattern => '[acdn]{1,4}',
 439         default => 'cdn',
 440     },
 441     bootdisk => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-bootdisk',
 444         description => "Enable booting from specified disk.",
 445         pattern => '(ide|sata|scsi|virtio)\d+',
 446     },
 447     smp => {
 448         optional => 1,
 449         type => 'integer',
 450         description => "The number of CPUs. Please use option -sockets instead.",
 451         minimum => 1,
 452         default => 1,
 453     },
 454     sockets => {
 455         optional => 1,
 456         type => 'integer',
 457         description => "The number of CPU sockets.",
 458         minimum => 1,
 459         default => 1,
 460     },
 461     cores => {
 462         optional => 1,
 463         type => 'integer',
 464         description => "The number of cores per socket.",
 465         minimum => 1,
 466         default => 1,
 467     },
 468     numa => {
 469         optional => 1,
 470         type => 'boolean',
 471         description => "Enable/disable NUMA.",
 472         default => 0,
 473     },
 474     hugepages => {
 475         optional => 1,
 476         type => 'string',
 477         description => "Enable/disable hugepages memory.",
 478         enum => [qw(any 2 1024)],
 479     },
 480     vcpus => {
 481         optional => 1,
 482         type => 'integer',
 483         description => "Number of hotplugged vcpus.",
 484         minimum => 1,
 485         default => 0,
 486     },
 487     acpi => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Enable/disable ACPI.",
 491         default => 1,
 492     },
 493     agent => {
 494         optional => 1,
 495         description => "Enable/disable Qemu GuestAgent and its properties.",
 496         type => 'string',
 497         format => $agent_fmt,
 498     },
 499     kvm => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Enable/disable KVM hardware virtualization.",
 503         default => 1,
 504     },
 505     tdf => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable time drift fix.",
 509         default => 0,
 510     },
 511     localtime => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 515     },
 516     freeze => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 520     },
 521     vga => {
 522         optional => 1,
 523         type => 'string', format => $vga_fmt,
 524         description => "Configure the VGA hardware.",
 525         verbose_description => "Configure the VGA Hardware. If you want to use ".
 526             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 527             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 528             "is 'std' for all OS types besides some Windows versions (XP and " .
 529             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 530             "display server. For win* OS you can select how many independent " .
 531             "displays you want, Linux guests can add displays them self.\n".
 532             "You can also run without any graphic card, using a serial device as terminal.",
 533     },
 534     watchdog => {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-watchdog',
 537         description => "Create a virtual hardware watchdog device.",
 538         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 539             " (by a guest action), the watchdog must be periodically polled " .
 540             "by an agent inside the guest or else the watchdog will reset " .
 541             "the guest (or execute the respective action specified)",
 542     },
 543     startdate => {
 544         optional => 1,
 545         type => 'string',
 546         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 547         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 548         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 549         default => 'now',
 550     },
 551     startup =>  get_standard_option('pve-startup-order'),
 552     template => {
 553         optional => 1,
 554         type => 'boolean',
 555         description => "Enable/disable Template.",
 556         default => 0,
 557     },
 558     args => {
 559         optional => 1,
 560         type => 'string',
 561         description => "Arbitrary arguments passed to kvm.",
 562         verbose_description => <<EODESCR,
 563 Arbitrary arguments passed to kvm, for example:
 564
 565 args: -no-reboot -no-hpet
 566
 567 NOTE: this option is for experts only.
 568 EODESCR
 569     },
 570     tablet => {
 571         optional => 1,
 572         type => 'boolean',
 573         default => 1,
 574         description => "Enable/disable the USB tablet device.",
 575         verbose_description => "Enable/disable the USB tablet device. This device is " .
 576             "usually needed to allow absolute mouse positioning with VNC. " .
 577             "Else the mouse runs out of sync with normal VNC clients. " .
 578             "If you're running lots of console-only guests on one host, " .
 579             "you may consider disabling this to save some context switches. " .
 580             "This is turned off by default if you use spice (-vga=qxl).",
 581     },
 582     migrate_speed => {
 583         optional => 1,
 584         type => 'integer',
 585         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 586         minimum => 0,
 587         default => 0,
 588     },
 589     migrate_downtime => {
 590         optional => 1,
 591         type => 'number',
 592         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 593         minimum => 0,
 594         default => 0.1,
 595     },
 596     cdrom => {
 597         optional => 1,
 598         type => 'string', format => 'pve-qm-ide',
 599         typetext => '<volume>',
 600         description => "This is an alias for option -ide2",
 601     },
 602     cpu => {
 603         optional => 1,
 604         description => "Emulated CPU type.",
 605         type => 'string',
 606         format => $cpu_fmt,
 607     },
 608     parent => get_standard_option('pve-snapshot-name', {
 609         optional => 1,
 610         description => "Parent snapshot name. This is used internally, and should not be modified.",
 611     }),
 612     snaptime => {
 613         optional => 1,
 614         description => "Timestamp for snapshots.",
 615         type => 'integer',
 616         minimum => 0,
 617     },
 618     vmstate => {
 619         optional => 1,
 620         type => 'string', format => 'pve-volume-id',
 621         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 622     },
 623     vmstatestorage => get_standard_option('pve-storage-id', {
 624         description => "Default storage for VM state volumes/files.",
 625         optional => 1,
 626     }),
 627     runningmachine => get_standard_option('pve-qemu-machine', {
 628         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 629     }),
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 647         default => 0,
 648     },
 649     bios => {
 650         optional => 1,
 651         type => 'string',
 652         enum => [ qw(seabios ovmf) ],
 653         description => "Select BIOS implementation.",
 654         default => 'seabios',
 655     },
 656     vmgenid => {
 657         type => 'string',
 658         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 659         format_description => 'UUID',
 660         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 662             " 128-bit integer value identifier to the guest OS. This allows to".
 663             " notify the guest operating system when the virtual machine is".
 664             " executed with a different configuration (e.g. snapshot execution".
 665             " or creation from a template). The guest operating system notices".
 666             " the change, and is then able to react as appropriate by marking".
 667             " its copies of distributed databases as dirty, re-initializing its".
 668             " random number generator, etc.\n".
 669             "Note that auto-creation only works when done throug API/CLI create".
 670             " or update methods, but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 684         optional => 1,
 685     },
 686     audio0 => {
 687         type => 'string',
 688         format => $audio_fmt,
 689         description => "Configure a audio device, useful in combination with QXL/Spice.",
 690         optional => 1
 691     },
 692     spice_enhancements => {
 693         type => 'string',
 694         format => $spice_enhancements_fmt,
 695         description => "Configure additional enhancements for SPICE.",
 696         optional => 1
 697     },
 698 };
 699
 700 my $cicustom_fmt = {
 701     meta => {
 702         type => 'string',
 703         optional => 1,
 704         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     network => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 712         format => 'pve-volume-id',
 713         format_description => 'volume',
 714     },
 715     user => {
 716         type => 'string',
 717         optional => 1,
 718         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 719         format => 'pve-volume-id',
 720         format_description => 'volume',
 721     },
 722 };
 723 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 724
 725 my $confdesc_cloudinit = {
 726     citype => {
 727         optional => 1,
 728         type => 'string',
 729         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 730         enum => ['configdrive2', 'nocloud'],
 731     },
 732     ciuser => {
 733         optional => 1,
 734         type => 'string',
 735         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 736     },
 737     cipassword => {
 738         optional => 1,
 739         type => 'string',
 740         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 741     },
 742     cicustom => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     sshkeys => {
 759         optional => 1,
 760         type => 'string',
 761         format => 'urlencoded',
 762         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 763     },
 764 };
 765
 766 # what about other qemu settings ?
 767 #cpu => 'string',
 768 #machine => 'string',
 769 #fda => 'file',
 770 #fdb => 'file',
 771 #mtdblock => 'file',
 772 #sd => 'file',
 773 #pflash => 'file',
 774 #snapshot => 'bool',
 775 #bootp => 'file',
 776 ##tftp => 'dir',
 777 ##smb => 'dir',
 778 #kernel => 'file',
 779 #append => 'string',
 780 #initrd => 'file',
 781 ##soundhw => 'string',
 782
 783 while (my ($k, $v) = each %$confdesc) {
 784     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 785 }
 786
 787 my $MAX_IDE_DISKS = 4;
 788 my $MAX_SCSI_DISKS = 14;
 789 my $MAX_VIRTIO_DISKS = 16;
 790 my $MAX_SATA_DISKS = 6;
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_UNUSED_DISKS = 256;
 794 my $MAX_HOSTPCI_DEVICES = 16;
 795 my $MAX_SERIAL_PORTS = 4;
 796 my $MAX_PARALLEL_PORTS = 3;
 797 my $MAX_NUMA = 8;
 798
 799 my $numa_fmt = {
 800     cpus => {
 801         type => "string",
 802         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 803         description => "CPUs accessing this NUMA node.",
 804         format_description => "id[-id];...",
 805     },
 806     memory => {
 807         type => "number",
 808         description => "Amount of memory this NUMA node provides.",
 809         optional => 1,
 810     },
 811     hostnodes => {
 812         type => "string",
 813         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 814         description => "Host NUMA nodes to use.",
 815         format_description => "id[-id];...",
 816         optional => 1,
 817     },
 818     policy => {
 819         type => 'string',
 820         enum => [qw(preferred bind interleave)],
 821         description => "NUMA allocation policy.",
 822         optional => 1,
 823     },
 824 };
 825 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 826 my $numadesc = {
 827     optional => 1,
 828     type => 'string', format => $numa_fmt,
 829     description => "NUMA topology.",
 830 };
 831 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 832
 833 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 834     $confdesc->{"numa$i"} = $numadesc;
 835 }
 836
 837 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 838                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 839                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 840 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 841
 842 my $net_fmt_bridge_descr = <<__EOD__;
 843 Bridge to attach the network device to. The Proxmox VE standard bridge
 844 is called 'vmbr0'.
 845
 846 If you do not specify a bridge, we create a kvm user (NATed) network
 847 device, which provides DHCP and DNS services. The following addresses
 848 are used:
 849
 850  10.0.2.2   Gateway
 851  10.0.2.3   DNS Server
 852  10.0.2.4   SMB Server
 853
 854 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 855 __EOD__
 856
 857 my $net_fmt = {
 858     macaddr  => get_standard_option('mac-addr', {
 859         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 860     }),
 861     model => {
 862         type => 'string',
 863         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 864         enum => $nic_model_list,
 865         default_key => 1,
 866     },
 867     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 868     bridge => {
 869         type => 'string',
 870         description => $net_fmt_bridge_descr,
 871         format_description => 'bridge',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909 };
 910
 911 my $netdesc = {
 912     optional => 1,
 913     type => 'string', format => $net_fmt,
 914     description => "Specify network devices.",
 915 };
 916
 917 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 918
 919 my $ipconfig_fmt = {
 920     ip => {
 921         type => 'string',
 922         format => 'pve-ipv4-config',
 923         format_description => 'IPv4Format/CIDR',
 924         description => 'IPv4 address in CIDR format.',
 925         optional => 1,
 926         default => 'dhcp',
 927     },
 928     gw => {
 929         type => 'string',
 930         format => 'ipv4',
 931         format_description => 'GatewayIPv4',
 932         description => 'Default gateway for IPv4 traffic.',
 933         optional => 1,
 934         requires => 'ip',
 935     },
 936     ip6 => {
 937         type => 'string',
 938         format => 'pve-ipv6-config',
 939         format_description => 'IPv6Format/CIDR',
 940         description => 'IPv6 address in CIDR format.',
 941         optional => 1,
 942         default => 'dhcp',
 943     },
 944     gw6 => {
 945         type => 'string',
 946         format => 'ipv6',
 947         format_description => 'GatewayIPv6',
 948         description => 'Default gateway for IPv6 traffic.',
 949         optional => 1,
 950         requires => 'ip6',
 951     },
 952 };
 953 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 954 my $ipconfigdesc = {
 955     optional => 1,
 956     type => 'string', format => 'pve-qm-ipconfig',
 957     description => <<'EODESCR',
 958 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 959
 960 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 961
 962 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 963 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 964
 965 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 966 EODESCR
 967 };
 968 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 969
 970 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 971     $confdesc->{"net$i"} = $netdesc;
 972     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 973 }
 974
 975 foreach my $key (keys %$confdesc_cloudinit) {
 976     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 977 }
 978
 979 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 980 sub verify_volume_id_or_qm_path {
 981     my ($volid, $noerr) = @_;
 982
 983     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 984         return $volid;
 985     }
 986
 987     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 988     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 989     if ($@) {
 990         return undef if $noerr;
 991         die $@;
 992     }
 993     return $volid;
 994 }
 995
 996 my $drivename_hash;
 997
 998 my %drivedesc_base = (
 999     volume => { alias => 'file' },
1000     file => {
1001         type => 'string',
1002         format => 'pve-volume-id-or-qm-path',
1003         default_key => 1,
1004         format_description => 'volume',
1005         description => "The drive's backing volume.",
1006     },
1007     media => {
1008         type => 'string',
1009         enum => [qw(cdrom disk)],
1010         description => "The drive's media type.",
1011         default => 'disk',
1012         optional => 1
1013     },
1014     cyls => {
1015         type => 'integer',
1016         description => "Force the drive's physical geometry to have a specific cylinder count.",
1017         optional => 1
1018     },
1019     heads => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific head count.",
1022         optional => 1
1023     },
1024     secs => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific sector count.",
1027         optional => 1
1028     },
1029     trans => {
1030         type => 'string',
1031         enum => [qw(none lba auto)],
1032         description => "Force disk geometry bios translation mode.",
1033         optional => 1,
1034     },
1035     snapshot => {
1036         type => 'boolean',
1037         description => "Controls qemu's snapshot mode feature."
1038             . " If activated, changes made to the disk are temporary and will"
1039             . " be discarded when the VM is shutdown.",
1040         optional => 1,
1041     },
1042     cache => {
1043         type => 'string',
1044         enum => [qw(none writethrough writeback unsafe directsync)],
1045         description => "The drive's cache mode",
1046         optional => 1,
1047     },
1048     format => get_standard_option('pve-qm-image-format'),
1049     size => {
1050         type => 'string',
1051         format => 'disk-size',
1052         format_description => 'DiskSize',
1053         description => "Disk size. This is purely informational and has no effect.",
1054         optional => 1,
1055     },
1056     backup => {
1057         type => 'boolean',
1058         description => "Whether the drive should be included when making backups.",
1059         optional => 1,
1060     },
1061     replicate => {
1062         type => 'boolean',
1063         description => 'Whether the drive should considered for replication jobs.',
1064         optional => 1,
1065         default => 1,
1066     },
1067     rerror => {
1068         type => 'string',
1069         enum => [qw(ignore report stop)],
1070         description => 'Read error action.',
1071         optional => 1,
1072     },
1073     werror => {
1074         type => 'string',
1075         enum => [qw(enospc ignore report stop)],
1076         description => 'Write error action.',
1077         optional => 1,
1078     },
1079     aio => {
1080         type => 'string',
1081         enum => [qw(native threads)],
1082         description => 'AIO type to use.',
1083         optional => 1,
1084     },
1085     discard => {
1086         type => 'string',
1087         enum => [qw(ignore on)],
1088         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1089         optional => 1,
1090     },
1091     detect_zeroes => {
1092         type => 'boolean',
1093         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1094         optional => 1,
1095     },
1096     serial => {
1097         type => 'string',
1098         format => 'urlencoded',
1099         format_description => 'serial',
1100         maxLength => 20*3, # *3 since it's %xx url enoded
1101         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1102         optional => 1,
1103     },
1104     shared => {
1105         type => 'boolean',
1106         description => 'Mark this locally-managed volume as available on all nodes',
1107         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1108         optional => 1,
1109         default => 0,
1110     }
1111 );
1112
1113 my %iothread_fmt = ( iothread => {
1114         type => 'boolean',
1115         description => "Whether to use iothreads for this drive",
1116         optional => 1,
1117 });
1118
1119 my %model_fmt = (
1120     model => {
1121         type => 'string',
1122         format => 'urlencoded',
1123         format_description => 'model',
1124         maxLength => 40*3, # *3 since it's %xx url enoded
1125         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1126         optional => 1,
1127     },
1128 );
1129
1130 my %queues_fmt = (
1131     queues => {
1132         type => 'integer',
1133         description => "Number of queues.",
1134         minimum => 2,
1135         optional => 1
1136     }
1137 );
1138
1139 my %scsiblock_fmt = (
1140     scsiblock => {
1141         type => 'boolean',
1142         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1143         optional => 1,
1144         default => 0,
1145     },
1146 );
1147
1148 my %ssd_fmt = (
1149     ssd => {
1150         type => 'boolean',
1151         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1152         optional => 1,
1153     },
1154 );
1155
1156 my %wwn_fmt = (
1157     wwn => {
1158         type => 'string',
1159         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1160         format_description => 'wwn',
1161         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1162         optional => 1,
1163     },
1164 );
1165
1166 my $add_throttle_desc = sub {
1167     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1168     my $d = {
1169         type => $type,
1170         format_description => $unit,
1171         description => "Maximum $what in $longunit.",
1172         optional => 1,
1173     };
1174     $d->{minimum} = $minimum if defined($minimum);
1175     $drivedesc_base{$key} = $d;
1176 };
1177 # throughput: (leaky bucket)
1178 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1179 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1180 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1181 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1182 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1183 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1184 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1185 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1186 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1187
1188 # pools: (pool of IO before throttling starts taking effect)
1189 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1190 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1192 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1193 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1194 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1195
1196 # burst lengths
1197 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1198 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1199 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1200 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1201 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1202 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203
1204 # legacy support
1205 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1206 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1207 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1208 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1209
1210 my $ide_fmt = {
1211     %drivedesc_base,
1212     %model_fmt,
1213     %ssd_fmt,
1214     %wwn_fmt,
1215 };
1216 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1217
1218 my $idedesc = {
1219     optional => 1,
1220     type => 'string', format => $ide_fmt,
1221     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1222 };
1223 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1224
1225 my $scsi_fmt = {
1226     %drivedesc_base,
1227     %iothread_fmt,
1228     %queues_fmt,
1229     %scsiblock_fmt,
1230     %ssd_fmt,
1231     %wwn_fmt,
1232 };
1233 my $scsidesc = {
1234     optional => 1,
1235     type => 'string', format => $scsi_fmt,
1236     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1237 };
1238 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1239
1240 my $sata_fmt = {
1241     %drivedesc_base,
1242     %ssd_fmt,
1243     %wwn_fmt,
1244 };
1245 my $satadesc = {
1246     optional => 1,
1247     type => 'string', format => $sata_fmt,
1248     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1249 };
1250 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1251
1252 my $virtio_fmt = {
1253     %drivedesc_base,
1254     %iothread_fmt,
1255 };
1256 my $virtiodesc = {
1257     optional => 1,
1258     type => 'string', format => $virtio_fmt,
1259     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1260 };
1261 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1262
1263 my $alldrive_fmt = {
1264     %drivedesc_base,
1265     %iothread_fmt,
1266     %model_fmt,
1267     %queues_fmt,
1268     %scsiblock_fmt,
1269     %ssd_fmt,
1270     %wwn_fmt,
1271 };
1272
1273 my $efidisk_fmt = {
1274     volume => { alias => 'file' },
1275     file => {
1276         type => 'string',
1277         format => 'pve-volume-id-or-qm-path',
1278         default_key => 1,
1279         format_description => 'volume',
1280         description => "The drive's backing volume.",
1281     },
1282     format => get_standard_option('pve-qm-image-format'),
1283     size => {
1284         type => 'string',
1285         format => 'disk-size',
1286         format_description => 'DiskSize',
1287         description => "Disk size. This is purely informational and has no effect.",
1288         optional => 1,
1289     },
1290 };
1291
1292 my $efidisk_desc = {
1293     optional => 1,
1294     type => 'string', format => $efidisk_fmt,
1295     description => "Configure a Disk for storing EFI vars",
1296 };
1297
1298 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1299
1300 my $usb_fmt = {
1301     host => {
1302         default_key => 1,
1303         type => 'string', format => 'pve-qm-usb-device',
1304         format_description => 'HOSTUSBDEVICE|spice',
1305         description => <<EODESCR,
1306 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1307
1308  'bus-port(.port)*' (decimal numbers) or
1309  'vendor_id:product_id' (hexadeciaml numbers) or
1310  'spice'
1311
1312 You can use the 'lsusb -t' command to list existing usb devices.
1313
1314 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1315
1316 The value 'spice' can be used to add a usb redirection devices for spice.
1317 EODESCR
1318     },
1319     usb3 => {
1320         optional => 1,
1321         type => 'boolean',
1322         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1323         default => 0,
1324     },
1325 };
1326
1327 my $usbdesc = {
1328     optional => 1,
1329     type => 'string', format => $usb_fmt,
1330     description => "Configure an USB device (n is 0 to 4).",
1331 };
1332 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1333
1334 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1335 my $hostpci_fmt = {
1336     host => {
1337         default_key => 1,
1338         type => 'string',
1339         pattern => qr/$PCIRE(;$PCIRE)*/,
1340         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1341         description => <<EODESCR,
1342 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1343 of PCI virtual functions of the host. HOSTPCIID syntax is:
1344
1345 'bus:dev.func' (hexadecimal numbers)
1346
1347 You can us the 'lspci' command to list existing PCI devices.
1348 EODESCR
1349     },
1350     rombar => {
1351         type => 'boolean',
1352         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1353         optional => 1,
1354         default => 1,
1355     },
1356     romfile => {
1357         type => 'string',
1358         pattern => '[^,;]+',
1359         format_description => 'string',
1360         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1361         optional => 1,
1362     },
1363     pcie => {
1364         type => 'boolean',
1365         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1366         optional => 1,
1367         default => 0,
1368     },
1369     'x-vga' => {
1370         type => 'boolean',
1371         description =>  "Enable vfio-vga device support.",
1372         optional => 1,
1373         default => 0,
1374     },
1375     'mdev' => {
1376         type => 'string',
1377         format_description => 'string',
1378         pattern => '[^/\.:]+',
1379         optional => 1,
1380         description => <<EODESCR
1381 The type of mediated device to use.
1382 An instance of this type will be created on startup of the VM and
1383 will be cleaned up when the VM stops.
1384 EODESCR
1385     }
1386 };
1387 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1388
1389 my $hostpcidesc = {
1390         optional => 1,
1391         type => 'string', format => 'pve-qm-hostpci',
1392         description => "Map host PCI devices into guest.",
1393         verbose_description =>  <<EODESCR,
1394 Map host PCI devices into guest.
1395
1396 NOTE: This option allows direct access to host hardware. So it is no longer
1397 possible to migrate such machines - use with special care.
1398
1399 CAUTION: Experimental! User reported problems with this option.
1400 EODESCR
1401 };
1402 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1403
1404 my $serialdesc = {
1405         optional => 1,
1406         type => 'string',
1407         pattern => '(/dev/.+|socket)',
1408         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1409         verbose_description =>  <<EODESCR,
1410 Create a serial device inside the VM (n is 0 to 3), and pass through a
1411 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1412 host side (use 'qm terminal' to open a terminal connection).
1413
1414 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1415
1416 CAUTION: Experimental! User reported problems with this option.
1417 EODESCR
1418 };
1419
1420 my $paralleldesc= {
1421         optional => 1,
1422         type => 'string',
1423         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1424         description =>  "Map host parallel devices (n is 0 to 2).",
1425         verbose_description =>  <<EODESCR,
1426 Map host parallel devices (n is 0 to 2).
1427
1428 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1429
1430 CAUTION: Experimental! User reported problems with this option.
1431 EODESCR
1432 };
1433
1434 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1435     $confdesc->{"parallel$i"} = $paralleldesc;
1436 }
1437
1438 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1439     $confdesc->{"serial$i"} = $serialdesc;
1440 }
1441
1442 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1443     $confdesc->{"hostpci$i"} = $hostpcidesc;
1444 }
1445
1446 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1447     $drivename_hash->{"ide$i"} = 1;
1448     $confdesc->{"ide$i"} = $idedesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1452     $drivename_hash->{"sata$i"} = 1;
1453     $confdesc->{"sata$i"} = $satadesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1457     $drivename_hash->{"scsi$i"} = 1;
1458     $confdesc->{"scsi$i"} = $scsidesc ;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1462     $drivename_hash->{"virtio$i"} = 1;
1463     $confdesc->{"virtio$i"} = $virtiodesc;
1464 }
1465
1466 $drivename_hash->{efidisk0} = 1;
1467 $confdesc->{efidisk0} = $efidisk_desc;
1468
1469 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1470     $confdesc->{"usb$i"} = $usbdesc;
1471 }
1472
1473 my $unuseddesc = {
1474     optional => 1,
1475     type => 'string', format => 'pve-volume-id',
1476     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1477 };
1478
1479 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1480     $confdesc->{"unused$i"} = $unuseddesc;
1481 }
1482
1483 my $kvm_api_version = 0;
1484
1485 sub kvm_version {
1486     return $kvm_api_version if $kvm_api_version;
1487
1488     open my $fh, '<', '/dev/kvm'
1489         or return undef;
1490
1491     # 0xae00 => KVM_GET_API_VERSION
1492     $kvm_api_version = ioctl($fh, 0xae00, 0);
1493
1494     return $kvm_api_version;
1495 }
1496
1497 my $kvm_user_version = {};
1498 my $kvm_mtime = {};
1499
1500 sub kvm_user_version {
1501     my ($binary) = @_;
1502
1503     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1504     my $st = stat($binary);
1505
1506     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1507     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1508         $cachedmtime == $st->mtime;
1509
1510     $kvm_user_version->{$binary} = 'unknown';
1511     $kvm_mtime->{$binary} = $st->mtime;
1512
1513     my $code = sub {
1514         my $line = shift;
1515         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1516             $kvm_user_version->{$binary} = $2;
1517         }
1518     };
1519
1520     eval { run_command([$binary, '--version'], outfunc => $code); };
1521     warn $@ if $@;
1522
1523     return $kvm_user_version->{$binary};
1524
1525 }
1526
1527 sub kernel_has_vhost_net {
1528     return -c '/dev/vhost-net';
1529 }
1530
1531 sub valid_drive_names {
1532     # order is important - used to autoselect boot disk
1533     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1534             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1535             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1536             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1537             'efidisk0');
1538 }
1539
1540 sub is_valid_drivename {
1541     my $dev = shift;
1542
1543     return defined($drivename_hash->{$dev});
1544 }
1545
1546 sub option_exists {
1547     my $key = shift;
1548     return defined($confdesc->{$key});
1549 }
1550
1551 sub nic_models {
1552     return $nic_model_list;
1553 }
1554
1555 sub os_list_description {
1556
1557     return {
1558         other => 'Other',
1559         wxp => 'Windows XP',
1560         w2k => 'Windows 2000',
1561         w2k3 =>, 'Windows 2003',
1562         w2k8 => 'Windows 2008',
1563         wvista => 'Windows Vista',
1564         win7 => 'Windows 7',
1565         win8 => 'Windows 8/2012',
1566         win10 => 'Windows 10/2016',
1567         l24 => 'Linux 2.4',
1568         l26 => 'Linux 2.6',
1569     };
1570 }
1571
1572 my $cdrom_path;
1573
1574 sub get_cdrom_path {
1575
1576     return  $cdrom_path if $cdrom_path;
1577
1578     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1579     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1580     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1581 }
1582
1583 sub get_iso_path {
1584     my ($storecfg, $vmid, $cdrom) = @_;
1585
1586     if ($cdrom eq 'cdrom') {
1587         return get_cdrom_path();
1588     } elsif ($cdrom eq 'none') {
1589         return '';
1590     } elsif ($cdrom =~ m|^/|) {
1591         return $cdrom;
1592     } else {
1593         return PVE::Storage::path($storecfg, $cdrom);
1594     }
1595 }
1596
1597 # try to convert old style file names to volume IDs
1598 sub filename_to_volume_id {
1599     my ($vmid, $file, $media) = @_;
1600
1601      if (!($file eq 'none' || $file eq 'cdrom' ||
1602           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1603
1604         return undef if $file =~ m|/|;
1605
1606         if ($media && $media eq 'cdrom') {
1607             $file = "local:iso/$file";
1608         } else {
1609             $file = "local:$vmid/$file";
1610         }
1611     }
1612
1613     return $file;
1614 }
1615
1616 sub verify_media_type {
1617     my ($opt, $vtype, $media) = @_;
1618
1619     return if !$media;
1620
1621     my $etype;
1622     if ($media eq 'disk') {
1623         $etype = 'images';
1624     } elsif ($media eq 'cdrom') {
1625         $etype = 'iso';
1626     } else {
1627         die "internal error";
1628     }
1629
1630     return if ($vtype eq $etype);
1631
1632     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1633 }
1634
1635 sub cleanup_drive_path {
1636     my ($opt, $storecfg, $drive) = @_;
1637
1638     # try to convert filesystem paths to volume IDs
1639
1640     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1641         ($drive->{file} !~ m|^/dev/.+|) &&
1642         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1643         ($drive->{file} !~ m/^\d+$/)) {
1644         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1645         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1646         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1647         verify_media_type($opt, $vtype, $drive->{media});
1648         $drive->{file} = $volid;
1649     }
1650
1651     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1652 }
1653
1654 sub parse_hotplug_features {
1655     my ($data) = @_;
1656
1657     my $res = {};
1658
1659     return $res if $data eq '0';
1660
1661     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1662
1663     foreach my $feature (PVE::Tools::split_list($data)) {
1664         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1665             $res->{$1} = 1;
1666         } else {
1667             die "invalid hotplug feature '$feature'\n";
1668         }
1669     }
1670     return $res;
1671 }
1672
1673 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1674 sub pve_verify_hotplug_features {
1675     my ($value, $noerr) = @_;
1676
1677     return $value if parse_hotplug_features($value);
1678
1679     return undef if $noerr;
1680
1681     die "unable to parse hotplug option\n";
1682 }
1683
1684 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1685 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1686 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1687 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1688 #        [,iothread=on][,serial=serial][,model=model]
1689
1690 sub parse_drive {
1691     my ($key, $data) = @_;
1692
1693     my ($interface, $index);
1694
1695     if ($key =~ m/^([^\d]+)(\d+)$/) {
1696         $interface = $1;
1697         $index = $2;
1698     } else {
1699         return undef;
1700     }
1701
1702     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1703                                      : $confdesc->{$key}->{format};
1704     if (!$desc) {
1705         warn "invalid drive key: $key\n";
1706         return undef;
1707     }
1708     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1709     return undef if !$res;
1710     $res->{interface} = $interface;
1711     $res->{index} = $index;
1712
1713     my $error = 0;
1714     foreach my $opt (qw(bps bps_rd bps_wr)) {
1715         if (my $bps = defined(delete $res->{$opt})) {
1716             if (defined($res->{"m$opt"})) {
1717                 warn "both $opt and m$opt specified\n";
1718                 ++$error;
1719                 next;
1720             }
1721             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1722         }
1723     }
1724
1725     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1726     for my $requirement (
1727         [mbps_max => 'mbps'],
1728         [mbps_rd_max => 'mbps_rd'],
1729         [mbps_wr_max => 'mbps_wr'],
1730         [miops_max => 'miops'],
1731         [miops_rd_max => 'miops_rd'],
1732         [miops_wr_max => 'miops_wr'],
1733         [bps_max_length => 'mbps_max'],
1734         [bps_rd_max_length => 'mbps_rd_max'],
1735         [bps_wr_max_length => 'mbps_wr_max'],
1736         [iops_max_length => 'iops_max'],
1737         [iops_rd_max_length => 'iops_rd_max'],
1738         [iops_wr_max_length => 'iops_wr_max']) {
1739         my ($option, $requires) = @$requirement;
1740         if ($res->{$option} && !$res->{$requires}) {
1741             warn "$option requires $requires\n";
1742             ++$error;
1743         }
1744     }
1745
1746     return undef if $error;
1747
1748     return undef if $res->{mbps_rd} && $res->{mbps};
1749     return undef if $res->{mbps_wr} && $res->{mbps};
1750     return undef if $res->{iops_rd} && $res->{iops};
1751     return undef if $res->{iops_wr} && $res->{iops};
1752
1753     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1754         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1755         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1756         return undef if $res->{interface} eq 'virtio';
1757     }
1758
1759     if (my $size = $res->{size}) {
1760         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1761     }
1762
1763     return $res;
1764 }
1765
1766 sub print_drive {
1767     my ($vmid, $drive) = @_;
1768     my $data = { %$drive };
1769     delete $data->{$_} for qw(index interface);
1770     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1771 }
1772
1773 sub scsi_inquiry {
1774     my($fh, $noerr) = @_;
1775
1776     my $SG_IO = 0x2285;
1777     my $SG_GET_VERSION_NUM = 0x2282;
1778
1779     my $versionbuf = "\x00" x 8;
1780     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1781     if (!$ret) {
1782         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1783         return undef;
1784     }
1785     my $version = unpack("I", $versionbuf);
1786     if ($version < 30000) {
1787         die "scsi generic interface too old\n"  if !$noerr;
1788         return undef;
1789     }
1790
1791     my $buf = "\x00" x 36;
1792     my $sensebuf = "\x00" x 8;
1793     my $cmd = pack("C x3 C x1", 0x12, 36);
1794
1795     # see /usr/include/scsi/sg.h
1796     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1797
1798     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1799                       length($sensebuf), 0, length($buf), $buf,
1800                       $cmd, $sensebuf, 6000);
1801
1802     $ret = ioctl($fh, $SG_IO, $packet);
1803     if (!$ret) {
1804         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1805         return undef;
1806     }
1807
1808     my @res = unpack($sg_io_hdr_t, $packet);
1809     if ($res[17] || $res[18]) {
1810         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1811         return undef;
1812     }
1813
1814     my $res = {};
1815     (my $byte0, my $byte1, $res->{vendor},
1816      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1817
1818     $res->{removable} = $byte1 & 128 ? 1 : 0;
1819     $res->{type} = $byte0 & 31;
1820
1821     return $res;
1822 }
1823
1824 sub path_is_scsi {
1825     my ($path) = @_;
1826
1827     my $fh = IO::File->new("+<$path") || return undef;
1828     my $res = scsi_inquiry($fh, 1);
1829     close($fh);
1830
1831     return $res;
1832 }
1833
1834 sub machine_type_is_q35 {
1835     my ($conf) = @_;
1836
1837     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1838 }
1839
1840 sub print_tabletdevice_full {
1841     my ($conf, $arch) = @_;
1842
1843     my $q35 = machine_type_is_q35($conf);
1844
1845     # we use uhci for old VMs because tablet driver was buggy in older qemu
1846     my $usbbus;
1847     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1848         $usbbus = 'ehci';
1849     } else {
1850         $usbbus = 'uhci';
1851     }
1852
1853     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1854 }
1855
1856 sub print_keyboarddevice_full {
1857     my ($conf, $arch, $machine) = @_;
1858
1859     return undef if $arch ne 'aarch64';
1860
1861     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1862 }
1863
1864 sub print_drivedevice_full {
1865     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1866
1867     my $device = '';
1868     my $maxdev = 0;
1869
1870     if ($drive->{interface} eq 'virtio') {
1871         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1872         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1873         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1874     } elsif ($drive->{interface} eq 'scsi') {
1875
1876         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1877         my $unit = $drive->{index} % $maxdev;
1878         my $devicetype = 'hd';
1879         my $path = '';
1880         if (drive_is_cdrom($drive)) {
1881             $devicetype = 'cd';
1882         } else {
1883             if ($drive->{file} =~ m|^/|) {
1884                 $path = $drive->{file};
1885                 if (my $info = path_is_scsi($path)) {
1886                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1887                         $devicetype = 'block';
1888                     } elsif ($info->{type} == 1) { # tape
1889                         $devicetype = 'generic';
1890                     }
1891                 }
1892             } else {
1893                  $path = PVE::Storage::path($storecfg, $drive->{file});
1894             }
1895
1896             if($path =~ m/^iscsi\:\/\//){
1897                 $devicetype = 'generic';
1898             }
1899         }
1900
1901         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1902             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1903         } else {
1904             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1905         }
1906
1907         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1908             $device .= ",rotation_rate=1";
1909         }
1910         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1911
1912     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1913         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1914         my $controller = int($drive->{index} / $maxdev);
1915         my $unit = $drive->{index} % $maxdev;
1916         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1917
1918         $device = "ide-$devicetype";
1919         if ($drive->{interface} eq 'ide') {
1920             $device .= ",bus=ide.$controller,unit=$unit";
1921         } else {
1922             $device .= ",bus=ahci$controller.$unit";
1923         }
1924         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1925
1926         if ($devicetype eq 'hd') {
1927             if (my $model = $drive->{model}) {
1928                 $model = URI::Escape::uri_unescape($model);
1929                 $device .= ",model=$model";
1930             }
1931             if ($drive->{ssd}) {
1932                 $device .= ",rotation_rate=1";
1933             }
1934         }
1935         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1936     } elsif ($drive->{interface} eq 'usb') {
1937         die "implement me";
1938         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1939     } else {
1940         die "unsupported interface type";
1941     }
1942
1943     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1944
1945     if (my $serial = $drive->{serial}) {
1946         $serial = URI::Escape::uri_unescape($serial);
1947         $device .= ",serial=$serial";
1948     }
1949
1950
1951     return $device;
1952 }
1953
1954 sub get_initiator_name {
1955     my $initiator;
1956
1957     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1958     while (defined(my $line = <$fh>)) {
1959         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1960         $initiator = $1;
1961         last;
1962     }
1963     $fh->close();
1964
1965     return $initiator;
1966 }
1967
1968 sub print_drive_full {
1969     my ($storecfg, $vmid, $drive) = @_;
1970
1971     my $path;
1972     my $volid = $drive->{file};
1973     my $format;
1974
1975     if (drive_is_cdrom($drive)) {
1976         $path = get_iso_path($storecfg, $vmid, $volid);
1977     } else {
1978         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1979         if ($storeid) {
1980             $path = PVE::Storage::path($storecfg, $volid);
1981             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1982             $format = qemu_img_format($scfg, $volname);
1983         } else {
1984             $path = $volid;
1985             $format = "raw";
1986         }
1987    }
1988
1989     my $opts = '';
1990     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1991     foreach my $o (@qemu_drive_options) {
1992         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1993     }
1994
1995     # snapshot only accepts on|off
1996     if (defined($drive->{snapshot})) {
1997         my $v = $drive->{snapshot} ? 'on' : 'off';
1998         $opts .= ",snapshot=$v";
1999     }
2000
2001     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2002         my ($dir, $qmpname) = @$type;
2003         if (my $v = $drive->{"mbps$dir"}) {
2004             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2005         }
2006         if (my $v = $drive->{"mbps${dir}_max"}) {
2007             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2008         }
2009         if (my $v = $drive->{"bps${dir}_max_length"}) {
2010             $opts .= ",throttling.bps$qmpname-max-length=$v";
2011         }
2012         if (my $v = $drive->{"iops${dir}"}) {
2013             $opts .= ",throttling.iops$qmpname=$v";
2014         }
2015         if (my $v = $drive->{"iops${dir}_max"}) {
2016             $opts .= ",throttling.iops$qmpname-max=$v";
2017         }
2018         if (my $v = $drive->{"iops${dir}_max_length"}) {
2019             $opts .= ",throttling.iops$qmpname-max-length=$v";
2020         }
2021     }
2022
2023     $opts .= ",format=$format" if $format && !$drive->{format};
2024
2025     my $cache_direct = 0;
2026
2027     if (my $cache = $drive->{cache}) {
2028         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2029     } elsif (!drive_is_cdrom($drive)) {
2030         $opts .= ",cache=none";
2031         $cache_direct = 1;
2032     }
2033
2034     # aio native works only with O_DIRECT
2035     if (!$drive->{aio}) {
2036         if($cache_direct) {
2037             $opts .= ",aio=native";
2038         } else {
2039             $opts .= ",aio=threads";
2040         }
2041     }
2042
2043     if (!drive_is_cdrom($drive)) {
2044         my $detectzeroes;
2045         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2046             $detectzeroes = 'off';
2047         } elsif ($drive->{discard}) {
2048             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2049         } else {
2050             # This used to be our default with discard not being specified:
2051             $detectzeroes = 'on';
2052         }
2053         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2054     }
2055
2056     my $pathinfo = $path ? "file=$path," : '';
2057
2058     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2059 }
2060
2061 sub print_netdevice_full {
2062     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2063
2064     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2065
2066     my $device = $net->{model};
2067     if ($net->{model} eq 'virtio') {
2068          $device = 'virtio-net-pci';
2069      };
2070
2071     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2072     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2073     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2074         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2075         my $vectors = $net->{queues} * 2 + 2;
2076         $tmpstr .= ",vectors=$vectors,mq=on";
2077     }
2078     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2079
2080     if ($use_old_bios_files) {
2081         my $romfile;
2082         if ($device eq 'virtio-net-pci') {
2083             $romfile = 'pxe-virtio.rom';
2084         } elsif ($device eq 'e1000') {
2085             $romfile = 'pxe-e1000.rom';
2086         } elsif ($device eq 'ne2k') {
2087             $romfile = 'pxe-ne2k_pci.rom';
2088         } elsif ($device eq 'pcnet') {
2089             $romfile = 'pxe-pcnet.rom';
2090         } elsif ($device eq 'rtl8139') {
2091             $romfile = 'pxe-rtl8139.rom';
2092         }
2093         $tmpstr .= ",romfile=$romfile" if $romfile;
2094     }
2095
2096     return $tmpstr;
2097 }
2098
2099 sub print_netdev_full {
2100     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2101
2102     my $i = '';
2103     if ($netid =~ m/^net(\d+)$/) {
2104         $i = int($1);
2105     }
2106
2107     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2108
2109     my $ifname = "tap${vmid}i$i";
2110
2111     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2112     die "interface name '$ifname' is too long (max 15 character)\n"
2113         if length($ifname) >= 16;
2114
2115     my $vhostparam = '';
2116     if (is_native($arch)) {
2117         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2118     }
2119
2120     my $vmname = $conf->{name} || "vm$vmid";
2121
2122     my $netdev = "";
2123     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2124
2125     if ($net->{bridge}) {
2126         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2127     } else {
2128         $netdev = "type=user,id=$netid,hostname=$vmname";
2129     }
2130
2131     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2132
2133     return $netdev;
2134 }
2135
2136
2137 sub print_cpu_device {
2138     my ($conf, $id) = @_;
2139
2140     my $kvm = $conf->{kvm} // 1;
2141     my $cpu = $kvm ? "kvm64" : "qemu64";
2142     if (my $cputype = $conf->{cpu}) {
2143         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2144             or die "Cannot parse cpu description: $cputype\n";
2145         $cpu = $cpuconf->{cputype};
2146     }
2147
2148     my $cores = $conf->{cores} || 1;
2149
2150     my $current_core = ($id - 1) % $cores;
2151     my $current_socket = int(($id - 1 - $current_core)/$cores);
2152
2153     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2154 }
2155
2156 my $vga_map = {
2157     'cirrus' => 'cirrus-vga',
2158     'std' => 'VGA',
2159     'vmware' => 'vmware-svga',
2160     'virtio' => 'virtio-vga',
2161 };
2162
2163 sub print_vga_device {
2164     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2165
2166     my $type = $vga_map->{$vga->{type}};
2167     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2168         $type = 'virtio-gpu';
2169     }
2170     my $vgamem_mb = $vga->{memory};
2171     if ($qxlnum) {
2172         $type = $id ? 'qxl' : 'qxl-vga';
2173     }
2174     die "no devicetype for $vga->{type}\n" if !$type;
2175
2176     my $memory = "";
2177     if ($vgamem_mb) {
2178         if ($vga->{type} eq 'virtio') {
2179             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2180             $memory = ",max_hostmem=$bytes";
2181         } elsif ($qxlnum) {
2182             # from https://www.spice-space.org/multiple-monitors.html
2183             $memory = ",vgamem_mb=$vga->{memory}";
2184             my $ram = $vgamem_mb * 4;
2185             my $vram = $vgamem_mb * 2;
2186             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2187         } else {
2188             $memory = ",vgamem_mb=$vga->{memory}";
2189         }
2190     } elsif ($qxlnum && $id) {
2191         $memory = ",ram_size=67108864,vram_size=33554432";
2192     }
2193
2194     my $q35 = machine_type_is_q35($conf);
2195     my $vgaid = "vga" . ($id // '');
2196     my $pciaddr;
2197
2198     if ($q35 && $vgaid eq 'vga') {
2199         # the first display uses pcie.0 bus on q35 machines
2200         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2201     } else {
2202         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2203     }
2204
2205     return "$type,id=${vgaid}${memory}${pciaddr}";
2206 }
2207
2208 sub drive_is_cloudinit {
2209     my ($drive) = @_;
2210     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2211 }
2212
2213 sub drive_is_cdrom {
2214     my ($drive, $exclude_cloudinit) = @_;
2215
2216     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2217
2218     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2219
2220 }
2221
2222 sub parse_number_sets {
2223     my ($set) = @_;
2224     my $res = [];
2225     foreach my $part (split(/;/, $set)) {
2226         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2227             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2228             push @$res, [ $1, $2 ];
2229         } else {
2230             die "invalid range: $part\n";
2231         }
2232     }
2233     return $res;
2234 }
2235
2236 sub parse_numa {
2237     my ($data) = @_;
2238
2239     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2240     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2241     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2242     return $res;
2243 }
2244
2245 sub parse_hostpci {
2246     my ($value) = @_;
2247
2248     return undef if !$value;
2249
2250     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2251
2252     my @idlist = split(/;/, $res->{host});
2253     delete $res->{host};
2254     foreach my $id (@idlist) {
2255         if ($id =~ m/\./) { # full id 00:00.1
2256             push @{$res->{pciid}}, {
2257                 id => $id,
2258             };
2259         } else { # partial id 00:00
2260             $res->{pciid} = PVE::SysFSTools::lspci($id);
2261         }
2262     }
2263     return $res;
2264 }
2265
2266 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2267 sub parse_net {
2268     my ($data) = @_;
2269
2270     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2271     if ($@) {
2272         warn $@;
2273         return undef;
2274     }
2275     if (!defined($res->{macaddr})) {
2276         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2277         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2278     }
2279     return $res;
2280 }
2281
2282 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2283 sub parse_ipconfig {
2284     my ($data) = @_;
2285
2286     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2287     if ($@) {
2288         warn $@;
2289         return undef;
2290     }
2291
2292     if ($res->{gw} && !$res->{ip}) {
2293         warn 'gateway specified without specifying an IP address';
2294         return undef;
2295     }
2296     if ($res->{gw6} && !$res->{ip6}) {
2297         warn 'IPv6 gateway specified without specifying an IPv6 address';
2298         return undef;
2299     }
2300     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2301         warn 'gateway specified together with DHCP';
2302         return undef;
2303     }
2304     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2305         # gw6 + auto/dhcp
2306         warn "IPv6 gateway specified together with $res->{ip6} address";
2307         return undef;
2308     }
2309
2310     if (!$res->{ip} && !$res->{ip6}) {
2311         return { ip => 'dhcp', ip6 => 'dhcp' };
2312     }
2313
2314     return $res;
2315 }
2316
2317 sub print_net {
2318     my $net = shift;
2319
2320     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2321 }
2322
2323 sub add_random_macs {
2324     my ($settings) = @_;
2325
2326     foreach my $opt (keys %$settings) {
2327         next if $opt !~ m/^net(\d+)$/;
2328         my $net = parse_net($settings->{$opt});
2329         next if !$net;
2330         $settings->{$opt} = print_net($net);
2331     }
2332 }
2333
2334 sub vm_is_volid_owner {
2335     my ($storecfg, $vmid, $volid) = @_;
2336
2337     if ($volid !~  m|^/|) {
2338         my ($path, $owner);
2339         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2340         if ($owner && ($owner == $vmid)) {
2341             return 1;
2342         }
2343     }
2344
2345     return undef;
2346 }
2347
2348 sub split_flagged_list {
2349     my $text = shift || '';
2350     $text =~ s/[,;]/ /g;
2351     $text =~ s/^\s+//;
2352     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2353 }
2354
2355 sub join_flagged_list {
2356     my ($how, $lst) = @_;
2357     join $how, map { $lst->{$_} . $_ } keys %$lst;
2358 }
2359
2360 sub vmconfig_delete_pending_option {
2361     my ($conf, $key, $force) = @_;
2362
2363     delete $conf->{pending}->{$key};
2364     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2365     $pending_delete_hash->{$key} = $force ? '!' : '';
2366     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2367 }
2368
2369 sub vmconfig_undelete_pending_option {
2370     my ($conf, $key) = @_;
2371
2372     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2373     delete $pending_delete_hash->{$key};
2374
2375     if (%$pending_delete_hash) {
2376         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2377     } else {
2378         delete $conf->{pending}->{delete};
2379     }
2380 }
2381
2382 sub vmconfig_register_unused_drive {
2383     my ($storecfg, $vmid, $conf, $drive) = @_;
2384
2385     if (drive_is_cloudinit($drive)) {
2386         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2387         warn $@ if $@;
2388     } elsif (!drive_is_cdrom($drive)) {
2389         my $volid = $drive->{file};
2390         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2391             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2392         }
2393     }
2394 }
2395
2396 sub vmconfig_cleanup_pending {
2397     my ($conf) = @_;
2398
2399     # remove pending changes when nothing changed
2400     my $changes;
2401     foreach my $opt (keys %{$conf->{pending}}) {
2402         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2403             $changes = 1;
2404             delete $conf->{pending}->{$opt};
2405         }
2406     }
2407
2408     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2409     my $pending_delete_hash = {};
2410     while (my ($opt, $force) = each %$current_delete_hash) {
2411         if (defined($conf->{$opt})) {
2412             $pending_delete_hash->{$opt} = $force;
2413         } else {
2414             $changes = 1;
2415         }
2416     }
2417
2418     if (%$pending_delete_hash) {
2419         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2420     } else {
2421         delete $conf->{pending}->{delete};
2422     }
2423
2424     return $changes;
2425 }
2426
2427 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2428 my $smbios1_fmt = {
2429     uuid => {
2430         type => 'string',
2431         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2432         format_description => 'UUID',
2433         description => "Set SMBIOS1 UUID.",
2434         optional => 1,
2435     },
2436     version => {
2437         type => 'string',
2438         pattern => '[A-Za-z0-9+\/]+={0,2}',
2439         format_description => 'Base64 encoded string',
2440         description => "Set SMBIOS1 version.",
2441         optional => 1,
2442     },
2443     serial => {
2444         type => 'string',
2445         pattern => '[A-Za-z0-9+\/]+={0,2}',
2446         format_description => 'Base64 encoded string',
2447         description => "Set SMBIOS1 serial number.",
2448         optional => 1,
2449     },
2450     manufacturer => {
2451         type => 'string',
2452         pattern => '[A-Za-z0-9+\/]+={0,2}',
2453         format_description => 'Base64 encoded string',
2454         description => "Set SMBIOS1 manufacturer.",
2455         optional => 1,
2456     },
2457     product => {
2458         type => 'string',
2459         pattern => '[A-Za-z0-9+\/]+={0,2}',
2460         format_description => 'Base64 encoded string',
2461         description => "Set SMBIOS1 product ID.",
2462         optional => 1,
2463     },
2464     sku => {
2465         type => 'string',
2466         pattern => '[A-Za-z0-9+\/]+={0,2}',
2467         format_description => 'Base64 encoded string',
2468         description => "Set SMBIOS1 SKU string.",
2469         optional => 1,
2470     },
2471     family => {
2472         type => 'string',
2473         pattern => '[A-Za-z0-9+\/]+={0,2}',
2474         format_description => 'Base64 encoded string',
2475         description => "Set SMBIOS1 family string.",
2476         optional => 1,
2477     },
2478     base64 => {
2479         type => 'boolean',
2480         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2481         optional => 1,
2482     },
2483 };
2484
2485 sub parse_smbios1 {
2486     my ($data) = @_;
2487
2488     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2489     warn $@ if $@;
2490     return $res;
2491 }
2492
2493 sub print_smbios1 {
2494     my ($smbios1) = @_;
2495     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2496 }
2497
2498 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2499
2500 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2501 sub verify_bootdisk {
2502     my ($value, $noerr) = @_;
2503
2504     return $value if is_valid_drivename($value);
2505
2506     return undef if $noerr;
2507
2508     die "invalid boot disk '$value'\n";
2509 }
2510
2511 sub parse_watchdog {
2512     my ($value) = @_;
2513
2514     return undef if !$value;
2515
2516     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2517     warn $@ if $@;
2518     return $res;
2519 }
2520
2521 sub parse_guest_agent {
2522     my ($value) = @_;
2523
2524     return {} if !defined($value->{agent});
2525
2526     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2527     warn $@ if $@;
2528
2529     # if the agent is disabled ignore the other potentially set properties
2530     return {} if !$res->{enabled};
2531     return $res;
2532 }
2533
2534 sub parse_vga {
2535     my ($value) = @_;
2536
2537     return {} if !$value;
2538     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2539     warn $@ if $@;
2540     return $res;
2541 }
2542
2543 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2544 sub verify_usb_device {
2545     my ($value, $noerr) = @_;
2546
2547     return $value if parse_usb_device($value);
2548
2549     return undef if $noerr;
2550
2551     die "unable to parse usb device\n";
2552 }
2553
2554 # add JSON properties for create and set function
2555 sub json_config_properties {
2556     my $prop = shift;
2557
2558     foreach my $opt (keys %$confdesc) {
2559         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2560         $prop->{$opt} = $confdesc->{$opt};
2561     }
2562
2563     return $prop;
2564 }
2565
2566 # return copy of $confdesc_cloudinit to generate documentation
2567 sub cloudinit_config_properties {
2568
2569     return dclone($confdesc_cloudinit);
2570 }
2571
2572 sub check_type {
2573     my ($key, $value) = @_;
2574
2575     die "unknown setting '$key'\n" if !$confdesc->{$key};
2576
2577     my $type = $confdesc->{$key}->{type};
2578
2579     if (!defined($value)) {
2580         die "got undefined value\n";
2581     }
2582
2583     if ($value =~ m/[\n\r]/) {
2584         die "property contains a line feed\n";
2585     }
2586
2587     if ($type eq 'boolean') {
2588         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2589         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2590         die "type check ('boolean') failed - got '$value'\n";
2591     } elsif ($type eq 'integer') {
2592         return int($1) if $value =~ m/^(\d+)$/;
2593         die "type check ('integer') failed - got '$value'\n";
2594     } elsif ($type eq 'number') {
2595         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2596         die "type check ('number') failed - got '$value'\n";
2597     } elsif ($type eq 'string') {
2598         if (my $fmt = $confdesc->{$key}->{format}) {
2599             PVE::JSONSchema::check_format($fmt, $value);
2600             return $value;
2601         }
2602         $value =~ s/^\"(.*)\"$/$1/;
2603         return $value;
2604     } else {
2605         die "internal error"
2606     }
2607 }
2608
2609 sub touch_config {
2610     my ($vmid) = @_;
2611
2612     my $conf = PVE::QemuConfig->config_file($vmid);
2613     utime undef, undef, $conf;
2614 }
2615
2616 sub destroy_vm {
2617     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2618
2619     my $conffile = PVE::QemuConfig->config_file($vmid);
2620
2621     my $conf = PVE::QemuConfig->load_config($vmid);
2622
2623     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2624
2625     if ($conf->{template}) {
2626         # check if any base image is still used by a linked clone
2627         foreach_drive($conf, sub {
2628                 my ($ds, $drive) = @_;
2629
2630                 return if drive_is_cdrom($drive);
2631
2632                 my $volid = $drive->{file};
2633
2634                 return if !$volid || $volid =~ m|^/|;
2635
2636                 die "base volume '$volid' is still in use by linked cloned\n"
2637                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2638
2639         });
2640     }
2641
2642     # only remove disks owned by this VM
2643     foreach_drive($conf, sub {
2644         my ($ds, $drive) = @_;
2645
2646         return if drive_is_cdrom($drive, 1);
2647
2648         my $volid = $drive->{file};
2649
2650         return if !$volid || $volid =~ m|^/|;
2651
2652         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2653         return if !$path || !$owner || ($owner != $vmid);
2654
2655         eval {
2656             PVE::Storage::vdisk_free($storecfg, $volid);
2657         };
2658         warn "Could not remove disk '$volid', check manually: $@" if $@;
2659
2660     });
2661
2662     if ($keep_empty_config) {
2663         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2664     } else {
2665         unlink $conffile;
2666     }
2667
2668     # also remove unused disk
2669     eval {
2670         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2671
2672         eval {
2673             PVE::Storage::foreach_volid($dl, sub {
2674                 my ($volid, $sid, $volname, $d) = @_;
2675                 PVE::Storage::vdisk_free($storecfg, $volid);
2676             });
2677         };
2678         warn $@ if $@;
2679
2680     };
2681     warn $@ if $@;
2682 }
2683
2684 sub parse_vm_config {
2685     my ($filename, $raw) = @_;
2686
2687     return undef if !defined($raw);
2688
2689     my $res = {
2690         digest => Digest::SHA::sha1_hex($raw),
2691         snapshots => {},
2692         pending => {},
2693     };
2694
2695     $filename =~ m|/qemu-server/(\d+)\.conf$|
2696         || die "got strange filename '$filename'";
2697
2698     my $vmid = $1;
2699
2700     my $conf = $res;
2701     my $descr;
2702     my $section = '';
2703
2704     my @lines = split(/\n/, $raw);
2705     foreach my $line (@lines) {
2706         next if $line =~ m/^\s*$/;
2707
2708         if ($line =~ m/^\[PENDING\]\s*$/i) {
2709             $section = 'pending';
2710             if (defined($descr)) {
2711                 $descr =~ s/\s+$//;
2712                 $conf->{description} = $descr;
2713             }
2714             $descr = undef;
2715             $conf = $res->{$section} = {};
2716             next;
2717
2718         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2719             $section = $1;
2720             if (defined($descr)) {
2721                 $descr =~ s/\s+$//;
2722                 $conf->{description} = $descr;
2723             }
2724             $descr = undef;
2725             $conf = $res->{snapshots}->{$section} = {};
2726             next;
2727         }
2728
2729         if ($line =~ m/^\#(.*)\s*$/) {
2730             $descr = '' if !defined($descr);
2731             $descr .= PVE::Tools::decode_text($1) . "\n";
2732             next;
2733         }
2734
2735         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2736             $descr = '' if !defined($descr);
2737             $descr .= PVE::Tools::decode_text($2);
2738         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2739             $conf->{snapstate} = $1;
2740         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2741             my $key = $1;
2742             my $value = $2;
2743             $conf->{$key} = $value;
2744         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2745             my $value = $1;
2746             if ($section eq 'pending') {
2747                 $conf->{delete} = $value; # we parse this later
2748             } else {
2749                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2750             }
2751         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2752             my $key = $1;
2753             my $value = $2;
2754             eval { $value = check_type($key, $value); };
2755             if ($@) {
2756                 warn "vm $vmid - unable to parse value of '$key' - $@";
2757             } else {
2758                 $key = 'ide2' if $key eq 'cdrom';
2759                 my $fmt = $confdesc->{$key}->{format};
2760                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2761                     my $v = parse_drive($key, $value);
2762                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2763                         $v->{file} = $volid;
2764                         $value = print_drive($vmid, $v);
2765                     } else {
2766                         warn "vm $vmid - unable to parse value of '$key'\n";
2767                         next;
2768                     }
2769                 }
2770
2771                 $conf->{$key} = $value;
2772             }
2773         }
2774     }
2775
2776     if (defined($descr)) {
2777         $descr =~ s/\s+$//;
2778         $conf->{description} = $descr;
2779     }
2780     delete $res->{snapstate}; # just to be sure
2781
2782     return $res;
2783 }
2784
2785 sub write_vm_config {
2786     my ($filename, $conf) = @_;
2787
2788     delete $conf->{snapstate}; # just to be sure
2789
2790     if ($conf->{cdrom}) {
2791         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2792         $conf->{ide2} = $conf->{cdrom};
2793         delete $conf->{cdrom};
2794     }
2795
2796     # we do not use 'smp' any longer
2797     if ($conf->{sockets}) {
2798         delete $conf->{smp};
2799     } elsif ($conf->{smp}) {
2800         $conf->{sockets} = $conf->{smp};
2801         delete $conf->{cores};
2802         delete $conf->{smp};
2803     }
2804
2805     my $used_volids = {};
2806
2807     my $cleanup_config = sub {
2808         my ($cref, $pending, $snapname) = @_;
2809
2810         foreach my $key (keys %$cref) {
2811             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2812                 $key eq 'snapstate' || $key eq 'pending';
2813             my $value = $cref->{$key};
2814             if ($key eq 'delete') {
2815                 die "propertry 'delete' is only allowed in [PENDING]\n"
2816                     if !$pending;
2817                 # fixme: check syntax?
2818                 next;
2819             }
2820             eval { $value = check_type($key, $value); };
2821             die "unable to parse value of '$key' - $@" if $@;
2822
2823             $cref->{$key} = $value;
2824
2825             if (!$snapname && is_valid_drivename($key)) {
2826                 my $drive = parse_drive($key, $value);
2827                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2828             }
2829         }
2830     };
2831
2832     &$cleanup_config($conf);
2833
2834     &$cleanup_config($conf->{pending}, 1);
2835
2836     foreach my $snapname (keys %{$conf->{snapshots}}) {
2837         die "internal error" if $snapname eq 'pending';
2838         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2839     }
2840
2841     # remove 'unusedX' settings if we re-add a volume
2842     foreach my $key (keys %$conf) {
2843         my $value = $conf->{$key};
2844         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2845             delete $conf->{$key};
2846         }
2847     }
2848
2849     my $generate_raw_config = sub {
2850         my ($conf, $pending) = @_;
2851
2852         my $raw = '';
2853
2854         # add description as comment to top of file
2855         if (defined(my $descr = $conf->{description})) {
2856             if ($descr) {
2857                 foreach my $cl (split(/\n/, $descr)) {
2858                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2859                 }
2860             } else {
2861                 $raw .= "#\n" if $pending;
2862             }
2863         }
2864
2865         foreach my $key (sort keys %$conf) {
2866             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2867             $raw .= "$key: $conf->{$key}\n";
2868         }
2869         return $raw;
2870     };
2871
2872     my $raw = &$generate_raw_config($conf);
2873
2874     if (scalar(keys %{$conf->{pending}})){
2875         $raw .= "\n[PENDING]\n";
2876         $raw .= &$generate_raw_config($conf->{pending}, 1);
2877     }
2878
2879     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2880         $raw .= "\n[$snapname]\n";
2881         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2882     }
2883
2884     return $raw;
2885 }
2886
2887 sub load_defaults {
2888
2889     my $res = {};
2890
2891     # we use static defaults from our JSON schema configuration
2892     foreach my $key (keys %$confdesc) {
2893         if (defined(my $default = $confdesc->{$key}->{default})) {
2894             $res->{$key} = $default;
2895         }
2896     }
2897
2898     return $res;
2899 }
2900
2901 sub config_list {
2902     my $vmlist = PVE::Cluster::get_vmlist();
2903     my $res = {};
2904     return $res if !$vmlist || !$vmlist->{ids};
2905     my $ids = $vmlist->{ids};
2906
2907     foreach my $vmid (keys %$ids) {
2908         my $d = $ids->{$vmid};
2909         next if !$d->{node} || $d->{node} ne $nodename;
2910         next if !$d->{type} || $d->{type} ne 'qemu';
2911         $res->{$vmid}->{exists} = 1;
2912     }
2913     return $res;
2914 }
2915
2916 # test if VM uses local resources (to prevent migration)
2917 sub check_local_resources {
2918     my ($conf, $noerr) = @_;
2919
2920     my @loc_res = ();
2921
2922     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2923     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2924
2925     push @loc_res, "ivshmem" if $conf->{ivshmem};
2926
2927     foreach my $k (keys %$conf) {
2928         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2929         # sockets are safe: they will recreated be on the target side post-migrate
2930         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2931         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2932     }
2933
2934     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2935
2936     return \@loc_res;
2937 }
2938
2939 # check if used storages are available on all nodes (use by migrate)
2940 sub check_storage_availability {
2941     my ($storecfg, $conf, $node) = @_;
2942
2943     foreach_drive($conf, sub {
2944         my ($ds, $drive) = @_;
2945
2946         my $volid = $drive->{file};
2947         return if !$volid;
2948
2949         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2950         return if !$sid;
2951
2952         # check if storage is available on both nodes
2953         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2954         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2955    });
2956 }
2957
2958 # list nodes where all VM images are available (used by has_feature API)
2959 sub shared_nodes {
2960     my ($conf, $storecfg) = @_;
2961
2962     my $nodelist = PVE::Cluster::get_nodelist();
2963     my $nodehash = { map { $_ => 1 } @$nodelist };
2964     my $nodename = PVE::INotify::nodename();
2965
2966     foreach_drive($conf, sub {
2967         my ($ds, $drive) = @_;
2968
2969         my $volid = $drive->{file};
2970         return if !$volid;
2971
2972         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2973         if ($storeid) {
2974             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2975             if ($scfg->{disable}) {
2976                 $nodehash = {};
2977             } elsif (my $avail = $scfg->{nodes}) {
2978                 foreach my $node (keys %$nodehash) {
2979                     delete $nodehash->{$node} if !$avail->{$node};
2980                 }
2981             } elsif (!$scfg->{shared}) {
2982                 foreach my $node (keys %$nodehash) {
2983                     delete $nodehash->{$node} if $node ne $nodename
2984                 }
2985             }
2986         }
2987     });
2988
2989     return $nodehash
2990 }
2991
2992 sub check_local_storage_availability {
2993     my ($conf, $storecfg) = @_;
2994
2995     my $nodelist = PVE::Cluster::get_nodelist();
2996     my $nodehash = { map { $_ => {} } @$nodelist };
2997
2998     foreach_drive($conf, sub {
2999         my ($ds, $drive) = @_;
3000
3001         my $volid = $drive->{file};
3002         return if !$volid;
3003
3004         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3005         if ($storeid) {
3006             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3007
3008             if ($scfg->{disable}) {
3009                 foreach my $node (keys %$nodehash) {
3010                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3011                 }
3012             } elsif (my $avail = $scfg->{nodes}) {
3013                 foreach my $node (keys %$nodehash) {
3014                     if (!$avail->{$node}) {
3015                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3016                     }
3017                 }
3018             }
3019         }
3020     });
3021
3022     foreach my $node (values %$nodehash) {
3023         if (my $unavail = $node->{unavailable_storages}) {
3024             $node->{unavailable_storages} = [ sort keys %$unavail ];
3025         }
3026     }
3027
3028     return $nodehash
3029 }
3030
3031 sub check_cmdline {
3032     my ($pidfile, $pid) = @_;
3033
3034     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3035     if (defined($fh)) {
3036         my $line = <$fh>;
3037         $fh->close;
3038         return undef if !$line;
3039         my @param = split(/\0/, $line);
3040
3041         my $cmd = $param[0];
3042         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3043
3044         for (my $i = 0; $i < scalar (@param); $i++) {
3045             my $p = $param[$i];
3046             next if !$p;
3047             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3048                 my $p = $param[$i+1];
3049                 return 1 if $p && ($p eq $pidfile);
3050                 return undef;
3051             }
3052         }
3053     }
3054     return undef;
3055 }
3056
3057 sub check_running {
3058     my ($vmid, $nocheck, $node) = @_;
3059
3060     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3061
3062     die "unable to find configuration file for VM $vmid - no such machine\n"
3063         if !$nocheck && ! -f $filename;
3064
3065     my $pidfile = pidfile_name($vmid);
3066
3067     if (my $fd = IO::File->new("<$pidfile")) {
3068         my $st = stat($fd);
3069         my $line = <$fd>;
3070         close($fd);
3071
3072         my $mtime = $st->mtime;
3073         if ($mtime > time()) {
3074             warn "file '$filename' modified in future\n";
3075         }
3076
3077         if ($line =~ m/^(\d+)$/) {
3078             my $pid = $1;
3079             if (check_cmdline($pidfile, $pid)) {
3080                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3081                     return $pid;
3082                 }
3083             }
3084         }
3085     }
3086
3087     return undef;
3088 }
3089
3090 sub vzlist {
3091
3092     my $vzlist = config_list();
3093
3094     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3095
3096     while (defined(my $de = $fd->read)) {
3097         next if $de !~ m/^(\d+)\.pid$/;
3098         my $vmid = $1;
3099         next if !defined($vzlist->{$vmid});
3100         if (my $pid = check_running($vmid)) {
3101             $vzlist->{$vmid}->{pid} = $pid;
3102         }
3103     }
3104
3105     return $vzlist;
3106 }
3107
3108 sub disksize {
3109     my ($storecfg, $conf) = @_;
3110
3111     my $bootdisk = $conf->{bootdisk};
3112     return undef if !$bootdisk;
3113     return undef if !is_valid_drivename($bootdisk);
3114
3115     return undef if !$conf->{$bootdisk};
3116
3117     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3118     return undef if !defined($drive);
3119
3120     return undef if drive_is_cdrom($drive);
3121
3122     my $volid = $drive->{file};
3123     return undef if !$volid;
3124
3125     return $drive->{size};
3126 }
3127
3128 our $vmstatus_return_properties = {
3129     vmid => get_standard_option('pve-vmid'),
3130     status => {
3131         description => "Qemu process status.",
3132         type => 'string',
3133         enum => ['stopped', 'running'],
3134     },
3135     maxmem => {
3136         description => "Maximum memory in bytes.",
3137         type => 'integer',
3138         optional => 1,
3139         renderer => 'bytes',
3140     },
3141     maxdisk => {
3142         description => "Root disk size in bytes.",
3143         type => 'integer',
3144         optional => 1,
3145         renderer => 'bytes',
3146     },
3147     name => {
3148         description => "VM name.",
3149         type => 'string',
3150         optional => 1,
3151     },
3152     qmpstatus => {
3153         description => "Qemu QMP agent status.",
3154         type => 'string',
3155         optional => 1,
3156     },
3157     pid => {
3158         description => "PID of running qemu process.",
3159         type => 'integer',
3160         optional => 1,
3161     },
3162     uptime => {
3163         description => "Uptime.",
3164         type => 'integer',
3165         optional => 1,
3166         renderer => 'duration',
3167     },
3168     cpus => {
3169         description => "Maximum usable CPUs.",
3170         type => 'number',
3171         optional => 1,
3172     },
3173     lock => {
3174         description => "The current config lock, if any.",
3175         type => 'string',
3176         optional => 1,
3177     }
3178 };
3179
3180 my $last_proc_pid_stat;
3181
3182 # get VM status information
3183 # This must be fast and should not block ($full == false)
3184 # We only query KVM using QMP if $full == true (this can be slow)
3185 sub vmstatus {
3186     my ($opt_vmid, $full) = @_;
3187
3188     my $res = {};
3189
3190     my $storecfg = PVE::Storage::config();
3191
3192     my $list = vzlist();
3193     my $defaults = load_defaults();
3194
3195     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3196
3197     my $cpucount = $cpuinfo->{cpus} || 1;
3198
3199     foreach my $vmid (keys %$list) {
3200         next if $opt_vmid && ($vmid ne $opt_vmid);
3201
3202         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3203         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3204
3205         my $d = { vmid => $vmid };
3206         $d->{pid} = $list->{$vmid}->{pid};
3207
3208         # fixme: better status?
3209         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3210
3211         my $size = disksize($storecfg, $conf);
3212         if (defined($size)) {
3213             $d->{disk} = 0; # no info available
3214             $d->{maxdisk} = $size;
3215         } else {
3216             $d->{disk} = 0;
3217             $d->{maxdisk} = 0;
3218         }
3219
3220         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3221             * ($conf->{cores} || $defaults->{cores});
3222         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3223         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3224
3225         $d->{name} = $conf->{name} || "VM $vmid";
3226         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3227             : $defaults->{memory}*(1024*1024);
3228
3229         if ($conf->{balloon}) {
3230             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3231             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3232                 : $defaults->{shares};
3233         }
3234
3235         $d->{uptime} = 0;
3236         $d->{cpu} = 0;
3237         $d->{mem} = 0;
3238
3239         $d->{netout} = 0;
3240         $d->{netin} = 0;
3241
3242         $d->{diskread} = 0;
3243         $d->{diskwrite} = 0;
3244
3245         $d->{template} = PVE::QemuConfig->is_template($conf);
3246
3247         $d->{serial} = 1 if conf_has_serial($conf);
3248         $d->{lock} = $conf->{lock} if $conf->{lock};
3249
3250         $res->{$vmid} = $d;
3251     }
3252
3253     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3254     foreach my $dev (keys %$netdev) {
3255         next if $dev !~ m/^tap([1-9]\d*)i/;
3256         my $vmid = $1;
3257         my $d = $res->{$vmid};
3258         next if !$d;
3259
3260         $d->{netout} += $netdev->{$dev}->{receive};
3261         $d->{netin} += $netdev->{$dev}->{transmit};
3262
3263         if ($full) {
3264             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3265             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3266         }
3267
3268     }
3269
3270     my $ctime = gettimeofday;
3271
3272     foreach my $vmid (keys %$list) {
3273
3274         my $d = $res->{$vmid};
3275         my $pid = $d->{pid};
3276         next if !$pid;
3277
3278         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3279         next if !$pstat; # not running
3280
3281         my $used = $pstat->{utime} + $pstat->{stime};
3282
3283         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3284
3285         if ($pstat->{vsize}) {
3286             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3287         }
3288
3289         my $old = $last_proc_pid_stat->{$pid};
3290         if (!$old) {
3291             $last_proc_pid_stat->{$pid} = {
3292                 time => $ctime,
3293                 used => $used,
3294                 cpu => 0,
3295             };
3296             next;
3297         }
3298
3299         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3300
3301         if ($dtime > 1000) {
3302             my $dutime = $used -  $old->{used};
3303
3304             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3305             $last_proc_pid_stat->{$pid} = {
3306                 time => $ctime,
3307                 used => $used,
3308                 cpu => $d->{cpu},
3309             };
3310         } else {
3311             $d->{cpu} = $old->{cpu};
3312         }
3313     }
3314
3315     return $res if !$full;
3316
3317     my $qmpclient = PVE::QMPClient->new();
3318
3319     my $ballooncb = sub {
3320         my ($vmid, $resp) = @_;
3321
3322         my $info = $resp->{'return'};
3323         return if !$info->{max_mem};
3324
3325         my $d = $res->{$vmid};
3326
3327         # use memory assigned to VM
3328         $d->{maxmem} = $info->{max_mem};
3329         $d->{balloon} = $info->{actual};
3330
3331         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3332             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3333             $d->{freemem} = $info->{free_mem};
3334         }
3335
3336         $d->{ballooninfo} = $info;
3337     };
3338
3339     my $blockstatscb = sub {
3340         my ($vmid, $resp) = @_;
3341         my $data = $resp->{'return'} || [];
3342         my $totalrdbytes = 0;
3343         my $totalwrbytes = 0;
3344
3345         for my $blockstat (@$data) {
3346             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3347             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3348
3349             $blockstat->{device} =~ s/drive-//;
3350             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3351         }
3352         $res->{$vmid}->{diskread} = $totalrdbytes;
3353         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3354     };
3355
3356     my $statuscb = sub {
3357         my ($vmid, $resp) = @_;
3358
3359         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3360         # this fails if ballon driver is not loaded, so this must be
3361         # the last commnand (following command are aborted if this fails).
3362         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3363
3364         my $status = 'unknown';
3365         if (!defined($status = $resp->{'return'}->{status})) {
3366             warn "unable to get VM status\n";
3367             return;
3368         }
3369
3370         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3371     };
3372
3373     foreach my $vmid (keys %$list) {
3374         next if $opt_vmid && ($vmid ne $opt_vmid);
3375         next if !$res->{$vmid}->{pid}; # not running
3376         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3377     }
3378
3379     $qmpclient->queue_execute(undef, 2);
3380
3381     foreach my $vmid (keys %$list) {
3382         next if $opt_vmid && ($vmid ne $opt_vmid);
3383         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3384     }
3385
3386     return $res;
3387 }
3388
3389 sub foreach_drive {
3390     my ($conf, $func, @param) = @_;
3391
3392     foreach my $ds (valid_drive_names()) {
3393         next if !defined($conf->{$ds});
3394
3395         my $drive = parse_drive($ds, $conf->{$ds});
3396         next if !$drive;
3397
3398         &$func($ds, $drive, @param);
3399     }
3400 }
3401
3402 sub foreach_volid {
3403     my ($conf, $func, @param) = @_;
3404
3405     my $volhash = {};
3406
3407     my $test_volid = sub {
3408         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3409
3410         return if !$volid;
3411
3412         $volhash->{$volid}->{cdrom} //= 1;
3413         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3414
3415         $volhash->{$volid}->{replicate} //= 0;
3416         $volhash->{$volid}->{replicate} = 1 if $replicate;
3417
3418         $volhash->{$volid}->{shared} //= 0;
3419         $volhash->{$volid}->{shared} = 1 if $shared;
3420
3421         $volhash->{$volid}->{referenced_in_config} //= 0;
3422         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3423
3424         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3425             if defined($snapname);
3426         $volhash->{$volid}->{size} = $size if $size;
3427     };
3428
3429     foreach_drive($conf, sub {
3430         my ($ds, $drive) = @_;
3431         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3432     });
3433
3434     foreach my $snapname (keys %{$conf->{snapshots}}) {
3435         my $snap = $conf->{snapshots}->{$snapname};
3436         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3437         foreach_drive($snap, sub {
3438             my ($ds, $drive) = @_;
3439             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3440         });
3441     }
3442
3443     foreach my $volid (keys %$volhash) {
3444         &$func($volid, $volhash->{$volid}, @param);
3445     }
3446 }
3447
3448 sub conf_has_serial {
3449     my ($conf) = @_;
3450
3451     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3452         if ($conf->{"serial$i"}) {
3453             return 1;
3454         }
3455     }
3456
3457     return 0;
3458 }
3459
3460 sub conf_has_audio {
3461     my ($conf, $id) = @_;
3462
3463     $id //= 0;
3464     my $audio = $conf->{"audio$id"};
3465     return undef if !defined($audio);
3466
3467     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3468     my $audiodriver = $audioproperties->{driver} // 'spice';
3469
3470     return {
3471         dev => $audioproperties->{device},
3472         dev_id => "audiodev$id",
3473         backend => $audiodriver,
3474         backend_id => "$audiodriver-backend${id}",
3475     };
3476 }
3477
3478 sub vga_conf_has_spice {
3479     my ($vga) = @_;
3480
3481     my $vgaconf = parse_vga($vga);
3482     my $vgatype = $vgaconf->{type};
3483     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3484
3485     return $1 || 1;
3486 }
3487
3488 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3489 sub get_host_arch() {
3490     $host_arch = (POSIX::uname())[4] if !$host_arch;
3491     return $host_arch;
3492 }
3493
3494 sub is_native($) {
3495     my ($arch) = @_;
3496     return get_host_arch() eq $arch;
3497 }
3498
3499 my $default_machines = {
3500     x86_64 => 'pc',
3501     aarch64 => 'virt',
3502 };
3503
3504 sub get_basic_machine_info {
3505     my ($conf, $forcemachine) = @_;
3506
3507     my $arch = $conf->{arch} // get_host_arch();
3508     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3509     return ($arch, $machine);
3510 }
3511
3512 sub get_ovmf_files($) {
3513     my ($arch) = @_;
3514
3515     my $ovmf = $OVMF->{$arch}
3516         or die "no OVMF images known for architecture '$arch'\n";
3517
3518     return @$ovmf;
3519 }
3520
3521 my $Arch2Qemu = {
3522     aarch64 => '/usr/bin/qemu-system-aarch64',
3523     x86_64 => '/usr/bin/qemu-system-x86_64',
3524 };
3525 sub get_command_for_arch($) {
3526     my ($arch) = @_;
3527     return '/usr/bin/kvm' if is_native($arch);
3528
3529     my $cmd = $Arch2Qemu->{$arch}
3530         or die "don't know how to emulate architecture '$arch'\n";
3531     return $cmd;
3532 }
3533
3534 sub get_cpu_options {
3535     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3536
3537     my $cpuFlags = [];
3538     my $ostype = $conf->{ostype};
3539
3540     my $cpu = $kvm ? "kvm64" : "qemu64";
3541     if ($arch eq 'aarch64') {
3542         $cpu = 'cortex-a57';
3543     }
3544     my $hv_vendor_id;
3545     if (my $cputype = $conf->{cpu}) {
3546         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3547             or die "Cannot parse cpu description: $cputype\n";
3548         $cpu = $cpuconf->{cputype};
3549         $kvm_off = 1 if $cpuconf->{hidden};
3550         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3551
3552         if (defined(my $flags = $cpuconf->{flags})) {
3553             push @$cpuFlags, split(";", $flags);
3554         }
3555     }
3556
3557     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3558
3559     push @$cpuFlags , '-x2apic'
3560         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3561
3562     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3563
3564     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3565
3566     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3567
3568         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3569         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3570     }
3571
3572     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3573
3574     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3575
3576     push @$cpuFlags, 'kvm=off' if $kvm_off;
3577
3578     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3579         push @$cpuFlags, "vendor=${cpu_vendor}"
3580             if $cpu_vendor ne 'default';
3581     } elsif ($arch ne 'aarch64') {
3582         die "internal error"; # should not happen
3583     }
3584
3585     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3586
3587     return ('-cpu', $cpu);
3588 }
3589
3590 sub config_to_command {
3591     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3592
3593     my $cmd = [];
3594     my $globalFlags = [];
3595     my $machineFlags = [];
3596     my $rtcFlags = [];
3597     my $devices = [];
3598     my $pciaddr = '';
3599     my $bridges = {};
3600     my $vernum = 0; # unknown
3601     my $ostype = $conf->{ostype};
3602     my $winversion = windows_version($ostype);
3603     my $kvm = $conf->{kvm};
3604
3605     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3606     my $kvm_binary = get_command_for_arch($arch);
3607     my $kvmver = kvm_user_version($kvm_binary);
3608     $kvm //= 1 if is_native($arch);
3609
3610     if ($kvm) {
3611         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3612             if !defined kvm_version();
3613     }
3614
3615     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3616         $vernum = $1*1000000+$2*1000;
3617     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3618         $vernum = $1*1000000+$2*1000+$3;
3619     }
3620
3621     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3622
3623     my $have_ovz = -f '/proc/vz/vestat';
3624
3625     my $q35 = machine_type_is_q35($conf);
3626     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3627     my $use_old_bios_files = undef;
3628     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3629
3630     my $cpuunits = defined($conf->{cpuunits}) ?
3631             $conf->{cpuunits} : $defaults->{cpuunits};
3632
3633     push @$cmd, $kvm_binary;
3634
3635     push @$cmd, '-id', $vmid;
3636
3637     my $vmname = $conf->{name} || "vm$vmid";
3638
3639     push @$cmd, '-name', $vmname;
3640
3641     my $use_virtio = 0;
3642
3643     my $qmpsocket = qmp_socket($vmid);
3644     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3645     push @$cmd, '-mon', "chardev=qmp,mode=control";
3646
3647     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3648         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3649         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3650     }
3651
3652     push @$cmd, '-pidfile' , pidfile_name($vmid);
3653
3654     push @$cmd, '-daemonize';
3655
3656     if ($conf->{smbios1}) {
3657         my $smbios_conf = parse_smbios1($conf->{smbios1});
3658         if ($smbios_conf->{base64}) {
3659             # Do not pass base64 flag to qemu
3660             delete $smbios_conf->{base64};
3661             my $smbios_string = "";
3662             foreach my $key (keys %$smbios_conf) {
3663                 my $value;
3664                 if ($key eq "uuid") {
3665                     $value = $smbios_conf->{uuid}
3666                 } else {
3667                     $value = decode_base64($smbios_conf->{$key});
3668                 }
3669                 # qemu accepts any binary data, only commas need escaping by double comma
3670                 $value =~ s/,/,,/g;
3671                 $smbios_string .= "," . $key . "=" . $value if $value;
3672             }
3673             push @$cmd, '-smbios', "type=1" . $smbios_string;
3674         } else {
3675             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3676         }
3677     }
3678
3679     if ($conf->{vmgenid}) {
3680         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3681     }
3682
3683     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3684     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3685         die "uefi base image not found\n" if ! -f $ovmf_code;
3686
3687         my $path;
3688         my $format;
3689         if (my $efidisk = $conf->{efidisk0}) {
3690             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3691             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3692             $format = $d->{format};
3693             if ($storeid) {
3694                 $path = PVE::Storage::path($storecfg, $d->{file});
3695                 if (!defined($format)) {
3696                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3697                     $format = qemu_img_format($scfg, $volname);
3698                 }
3699             } else {
3700                 $path = $d->{file};
3701                 die "efidisk format must be specified\n"
3702                     if !defined($format);
3703             }
3704         } else {
3705             warn "no efidisk configured! Using temporary efivars disk.\n";
3706             $path = "/tmp/$vmid-ovmf.fd";
3707             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3708             $format = 'raw';
3709         }
3710
3711         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3712         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3713     }
3714
3715     # load q35 config
3716     if ($q35) {
3717         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3718         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3719             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3720         } else {
3721             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3722         }
3723     }
3724
3725     # add usb controllers
3726     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3727     push @$devices, @usbcontrollers if @usbcontrollers;
3728     my $vga = parse_vga($conf->{vga});
3729
3730     my $qxlnum = vga_conf_has_spice($conf->{vga});
3731     $vga->{type} = 'qxl' if $qxlnum;
3732
3733     if (!$vga->{type}) {
3734         if ($arch eq 'aarch64') {
3735             $vga->{type} = 'virtio';
3736         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3737             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3738         } else {
3739             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3740         }
3741     }
3742
3743     # enable absolute mouse coordinates (needed by vnc)
3744     my $tablet;
3745     if (defined($conf->{tablet})) {
3746         $tablet = $conf->{tablet};
3747     } else {
3748         $tablet = $defaults->{tablet};
3749         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3750         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3751     }
3752
3753     if ($tablet) {
3754         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3755         my $kbd = print_keyboarddevice_full($conf, $arch);
3756         push @$devices, '-device', $kbd if defined($kbd);
3757     }
3758
3759     my $kvm_off = 0;
3760     my $gpu_passthrough;
3761
3762     # host pci devices
3763     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3764         my $id = "hostpci$i";
3765         my $d = parse_hostpci($conf->{$id});
3766         next if !$d;
3767
3768         if (my $pcie = $d->{pcie}) {
3769             die "q35 machine model is not enabled" if !$q35;
3770             # win7 wants to have the pcie devices directly on the pcie bus
3771             # instead of in the root port
3772             if ($winversion == 7) {
3773                 $pciaddr = print_pcie_addr("${id}bus0");
3774             } else {
3775                 # add more root ports if needed, 4 are present by default
3776                 # by pve-q35 cfgs, rest added here on demand.
3777                 if ($i > 3) {
3778                     push @$devices, '-device', print_pcie_root_port($i);
3779                 }
3780                 $pciaddr = print_pcie_addr($id);
3781             }
3782         } else {
3783             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3784         }
3785
3786         my $xvga = '';
3787         if ($d->{'x-vga'}) {
3788             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3789             $kvm_off = 1;
3790             $vga->{type} = 'none' if !defined($conf->{vga});
3791             $gpu_passthrough = 1;
3792         }
3793
3794         my $pcidevices = $d->{pciid};
3795         my $multifunction = 1 if @$pcidevices > 1;
3796
3797         my $sysfspath;
3798         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3799             my $pci_id = $pcidevices->[0]->{id};
3800             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3801             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3802         } elsif ($d->{mdev}) {
3803             warn "ignoring mediated device '$id' with multifunction device\n";
3804         }
3805
3806         my $j=0;
3807         foreach my $pcidevice (@$pcidevices) {
3808             my $devicestr = "vfio-pci";
3809
3810             if ($sysfspath) {
3811                 $devicestr .= ",sysfsdev=$sysfspath";
3812             } else {
3813                 $devicestr .= ",host=$pcidevice->{id}";
3814             }
3815
3816             my $mf_addr = $multifunction ? ".$j" : '';
3817             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3818
3819             if ($j == 0) {
3820                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3821                 $devicestr .= "$xvga";
3822                 $devicestr .= ",multifunction=on" if $multifunction;
3823                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3824             }
3825
3826             push @$devices, '-device', $devicestr;
3827             $j++;
3828         }
3829     }
3830
3831     # usb devices
3832     my $usb_dev_features = {};
3833     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 1);
3834
3835     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3836     push @$devices, @usbdevices if @usbdevices;
3837     # serial devices
3838     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3839         if (my $path = $conf->{"serial$i"}) {
3840             if ($path eq 'socket') {
3841                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3842                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3843                 # On aarch64, serial0 is the UART device. Qemu only allows
3844                 # connecting UART devices via the '-serial' command line, as
3845                 # the device has a fixed slot on the hardware...
3846                 if ($arch eq 'aarch64' && $i == 0) {
3847                     push @$devices, '-serial', "chardev:serial$i";
3848                 } else {
3849                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3850                 }
3851             } else {
3852                 die "no such serial device\n" if ! -c $path;
3853                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3854                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3855             }
3856         }
3857     }
3858
3859     # parallel devices
3860     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3861         if (my $path = $conf->{"parallel$i"}) {
3862             die "no such parallel device\n" if ! -c $path;
3863             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3864             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3865             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3866         }
3867     }
3868
3869     if (my $audio = conf_has_audio($conf)) {
3870
3871         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3872
3873         my $id = $audio->{dev_id};
3874         if ($audio->{dev} eq 'AC97') {
3875             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3876         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3877             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3878             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3879             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3880         } else {
3881             die "unkown audio device '$audio->{dev}', implement me!";
3882         }
3883
3884         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3885     }
3886
3887     my $sockets = 1;
3888     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3889     $sockets = $conf->{sockets} if  $conf->{sockets};
3890
3891     my $cores = $conf->{cores} || 1;
3892
3893     my $maxcpus = $sockets * $cores;
3894
3895     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3896
3897     my $allowed_vcpus = $cpuinfo->{cpus};
3898
3899     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3900         if ($allowed_vcpus < $maxcpus);
3901
3902     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3903
3904         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3905         for (my $i = 2; $i <= $vcpus; $i++)  {
3906             my $cpustr = print_cpu_device($conf,$i);
3907             push @$cmd, '-device', $cpustr;
3908         }
3909
3910     } else {
3911
3912         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3913     }
3914     push @$cmd, '-nodefaults';
3915
3916     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3917
3918     my $bootindex_hash = {};
3919     my $i = 1;
3920     foreach my $o (split(//, $bootorder)) {
3921         $bootindex_hash->{$o} = $i*100;
3922         $i++;
3923     }
3924
3925     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3926
3927     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3928
3929     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3930
3931     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3932         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3933         my $socket = vnc_socket($vmid);
3934         push @$cmd,  '-vnc', "unix:$socket,password";
3935     } else {
3936         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3937         push @$cmd, '-nographic';
3938     }
3939
3940     # time drift fix
3941     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3942
3943     my $useLocaltime = $conf->{localtime};
3944
3945     if ($winversion >= 5) { # windows
3946         $useLocaltime = 1 if !defined($conf->{localtime});
3947
3948         # use time drift fix when acpi is enabled
3949         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3950             $tdf = 1 if !defined($conf->{tdf});
3951         }
3952     }
3953
3954     if ($winversion >= 6) {
3955         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3956         push @$cmd, '-no-hpet';
3957     }
3958
3959     push @$rtcFlags, 'driftfix=slew' if $tdf;
3960
3961     if (!$kvm) {
3962         push @$machineFlags, 'accel=tcg';
3963     }
3964
3965     if ($machine_type) {
3966         push @$machineFlags, "type=${machine_type}";
3967     }
3968
3969     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3970         push @$rtcFlags, "base=$conf->{startdate}";
3971     } elsif ($useLocaltime) {
3972         push @$rtcFlags, 'base=localtime';
3973     }
3974
3975     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3976
3977     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3978
3979     push @$cmd, '-S' if $conf->{freeze};
3980
3981     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3982
3983     if (parse_guest_agent($conf)->{enabled}) {
3984         my $qgasocket = qmp_socket($vmid, 1);
3985         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3986         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3987         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3988         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3989     }
3990
3991     my $spice_port;
3992
3993     if ($qxlnum) {
3994         if ($qxlnum > 1) {
3995             if ($winversion){
3996                 for(my $i = 1; $i < $qxlnum; $i++){
3997                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3998                 }
3999             } else {
4000                 # assume other OS works like Linux
4001                 my ($ram, $vram) = ("134217728", "67108864");
4002                 if ($vga->{memory}) {
4003                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4004                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4005                 }
4006                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4007                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4008             }
4009         }
4010
4011         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4012
4013         my $nodename = PVE::INotify::nodename();
4014         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4015         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4016         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4017         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4018         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4019
4020         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4021         if ($spice_enhancement->{foldersharing}) {
4022             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4023             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4024         }
4025
4026         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4027         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4028         push @$devices, '-spice', "$spice_opts";
4029
4030         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4031         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4032         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4033
4034     }
4035
4036     # enable balloon by default, unless explicitly disabled
4037     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4038         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4039         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4040     }
4041
4042     if ($conf->{watchdog}) {
4043         my $wdopts = parse_watchdog($conf->{watchdog});
4044         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4045         my $watchdog = $wdopts->{model} || 'i6300esb';
4046         push @$devices, '-device', "$watchdog$pciaddr";
4047         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4048     }
4049
4050     my $vollist = [];
4051     my $scsicontroller = {};
4052     my $ahcicontroller = {};
4053     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4054
4055     # Add iscsi initiator name if available
4056     if (my $initiator = get_initiator_name()) {
4057         push @$devices, '-iscsi', "initiator-name=$initiator";
4058     }
4059
4060     foreach_drive($conf, sub {
4061         my ($ds, $drive) = @_;
4062
4063         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4064             push @$vollist, $drive->{file};
4065         }
4066
4067         # ignore efidisk here, already added in bios/fw handling code above
4068         return if $drive->{interface} eq 'efidisk';
4069
4070         $use_virtio = 1 if $ds =~ m/^virtio/;
4071
4072         if (drive_is_cdrom ($drive)) {
4073             if ($bootindex_hash->{d}) {
4074                 $drive->{bootindex} = $bootindex_hash->{d};
4075                 $bootindex_hash->{d} += 1;
4076             }
4077         } else {
4078             if ($bootindex_hash->{c}) {
4079                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4080                 $bootindex_hash->{c} += 1;
4081             }
4082         }
4083
4084         if($drive->{interface} eq 'virtio'){
4085            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4086         }
4087
4088         if ($drive->{interface} eq 'scsi') {
4089
4090             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4091
4092             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4093             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4094
4095             my $iothread = '';
4096             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4097                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4098                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4099             } elsif ($drive->{iothread}) {
4100                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4101             }
4102
4103             my $queues = '';
4104             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4105                 $queues = ",num_queues=$drive->{queues}";
4106             }
4107
4108             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4109             $scsicontroller->{$controller}=1;
4110         }
4111
4112         if ($drive->{interface} eq 'sata') {
4113            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4114            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4115            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4116            $ahcicontroller->{$controller}=1;
4117         }
4118
4119         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4120         push @$devices, '-drive',$drive_cmd;
4121         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4122     });
4123
4124     for (my $i = 0; $i < $MAX_NETS; $i++) {
4125          next if !$conf->{"net$i"};
4126          my $d = parse_net($conf->{"net$i"});
4127          next if !$d;
4128
4129          $use_virtio = 1 if $d->{model} eq 'virtio';
4130
4131          if ($bootindex_hash->{n}) {
4132             $d->{bootindex} = $bootindex_hash->{n};
4133             $bootindex_hash->{n} += 1;
4134          }
4135
4136          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4137          push @$devices, '-netdev', $netdevfull;
4138
4139          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4140          push @$devices, '-device', $netdevicefull;
4141     }
4142
4143     if ($conf->{ivshmem}) {
4144         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4145
4146         my $bus;
4147         if ($q35) {
4148             $bus = print_pcie_addr("ivshmem");
4149         } else {
4150             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4151         }
4152
4153         my $ivshmem_name = $ivshmem->{name} // $vmid;
4154         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4155
4156         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4157         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4158     }
4159
4160     if (!$q35) {
4161         # add pci bridges
4162         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4163            $bridges->{1} = 1;
4164            $bridges->{2} = 1;
4165         }
4166
4167         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4168
4169         while (my ($k, $v) = each %$bridges) {
4170             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4171             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4172         }
4173     }
4174
4175     push @$cmd, @$devices;
4176     push @$cmd, '-rtc', join(',', @$rtcFlags)
4177         if scalar(@$rtcFlags);
4178     push @$cmd, '-machine', join(',', @$machineFlags)
4179         if scalar(@$machineFlags);
4180     push @$cmd, '-global', join(',', @$globalFlags)
4181         if scalar(@$globalFlags);
4182
4183     if (my $vmstate = $conf->{vmstate}) {
4184         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4185         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4186         push @$cmd, '-loadstate', $statepath;
4187     }
4188
4189     # add custom args
4190     if ($conf->{args}) {
4191         my $aa = PVE::Tools::split_args($conf->{args});
4192         push @$cmd, @$aa;
4193     }
4194
4195     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4196 }
4197
4198 sub vnc_socket {
4199     my ($vmid) = @_;
4200     return "${var_run_tmpdir}/$vmid.vnc";
4201 }
4202
4203 sub spice_port {
4204     my ($vmid) = @_;
4205
4206     my $res = vm_mon_cmd($vmid, 'query-spice');
4207
4208     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4209 }
4210
4211 sub qmp_socket {
4212     my ($vmid, $qga, $name) = @_;
4213     my $sockettype = $qga ? 'qga' : 'qmp';
4214     my $ext = $name ? '-'.$name : '';
4215     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4216 }
4217
4218 sub pidfile_name {
4219     my ($vmid) = @_;
4220     return "${var_run_tmpdir}/$vmid.pid";
4221 }
4222
4223 sub vm_devices_list {
4224     my ($vmid) = @_;
4225
4226     my $res = vm_mon_cmd($vmid, 'query-pci');
4227     my $devices_to_check = [];
4228     my $devices = {};
4229     foreach my $pcibus (@$res) {
4230         push @$devices_to_check, @{$pcibus->{devices}},
4231     }
4232
4233     while (@$devices_to_check) {
4234         my $to_check = [];
4235         for my $d (@$devices_to_check) {
4236             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4237             next if !$d->{'pci_bridge'};
4238
4239             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4240             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4241         }
4242         $devices_to_check = $to_check;
4243     }
4244
4245     my $resblock = vm_mon_cmd($vmid, 'query-block');
4246     foreach my $block (@$resblock) {
4247         if($block->{device} =~ m/^drive-(\S+)/){
4248                 $devices->{$1} = 1;
4249         }
4250     }
4251
4252     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4253     foreach my $mice (@$resmice) {
4254         if ($mice->{name} eq 'QEMU HID Tablet') {
4255             $devices->{tablet} = 1;
4256             last;
4257         }
4258     }
4259
4260     # for usb devices there is no query-usb
4261     # but we can iterate over the entries in
4262     # qom-list path=/machine/peripheral
4263     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4264     foreach my $per (@$resperipheral) {
4265         if ($per->{name} =~ m/^usb\d+$/) {
4266             $devices->{$per->{name}} = 1;
4267         }
4268     }
4269
4270     return $devices;
4271 }
4272
4273 sub vm_deviceplug {
4274     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4275
4276     my $q35 = machine_type_is_q35($conf);
4277
4278     my $devices_list = vm_devices_list($vmid);
4279     return 1 if defined($devices_list->{$deviceid});
4280
4281     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4282
4283     if ($deviceid eq 'tablet') {
4284
4285         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4286
4287     } elsif ($deviceid eq 'keyboard') {
4288
4289         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4290
4291     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4292
4293         die "usb hotplug currently not reliable\n";
4294         # since we can't reliably hot unplug all added usb devices
4295         # and usb passthrough disables live migration
4296         # we disable usb hotplugging for now
4297         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4298
4299     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4300
4301         qemu_iothread_add($vmid, $deviceid, $device);
4302
4303         qemu_driveadd($storecfg, $vmid, $device);
4304         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4305
4306         qemu_deviceadd($vmid, $devicefull);
4307         eval { qemu_deviceaddverify($vmid, $deviceid); };
4308         if (my $err = $@) {
4309             eval { qemu_drivedel($vmid, $deviceid); };
4310             warn $@ if $@;
4311             die $err;
4312         }
4313
4314     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4315
4316
4317         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4318         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4319         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4320
4321         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4322
4323         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4324             qemu_iothread_add($vmid, $deviceid, $device);
4325             $devicefull .= ",iothread=iothread-$deviceid";
4326         }
4327
4328         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4329             $devicefull .= ",num_queues=$device->{queues}";
4330         }
4331
4332         qemu_deviceadd($vmid, $devicefull);
4333         qemu_deviceaddverify($vmid, $deviceid);
4334
4335     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4336
4337         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4338         qemu_driveadd($storecfg, $vmid, $device);
4339
4340         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4341         eval { qemu_deviceadd($vmid, $devicefull); };
4342         if (my $err = $@) {
4343             eval { qemu_drivedel($vmid, $deviceid); };
4344             warn $@ if $@;
4345             die $err;
4346         }
4347
4348     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4349
4350         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4351
4352         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4353         my $use_old_bios_files = undef;
4354         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4355
4356         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4357         qemu_deviceadd($vmid, $netdevicefull);
4358         eval {
4359             qemu_deviceaddverify($vmid, $deviceid);
4360             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4361         };
4362         if (my $err = $@) {
4363             eval { qemu_netdevdel($vmid, $deviceid); };
4364             warn $@ if $@;
4365             die $err;
4366         }
4367
4368     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4369
4370         my $bridgeid = $2;
4371         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4372         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4373
4374         qemu_deviceadd($vmid, $devicefull);
4375         qemu_deviceaddverify($vmid, $deviceid);
4376
4377     } else {
4378         die "can't hotplug device '$deviceid'\n";
4379     }
4380
4381     return 1;
4382 }
4383
4384 # fixme: this should raise exceptions on error!
4385 sub vm_deviceunplug {
4386     my ($vmid, $conf, $deviceid) = @_;
4387
4388     my $devices_list = vm_devices_list($vmid);
4389     return 1 if !defined($devices_list->{$deviceid});
4390
4391     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4392
4393     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4394
4395         qemu_devicedel($vmid, $deviceid);
4396
4397     } elsif ($deviceid =~ m/^usb\d+$/) {
4398
4399         die "usb hotplug currently not reliable\n";
4400         # when unplugging usb devices this way,
4401         # there may be remaining usb controllers/hubs
4402         # so we disable it for now
4403         qemu_devicedel($vmid, $deviceid);
4404         qemu_devicedelverify($vmid, $deviceid);
4405
4406     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4407
4408         qemu_devicedel($vmid, $deviceid);
4409         qemu_devicedelverify($vmid, $deviceid);
4410         qemu_drivedel($vmid, $deviceid);
4411         qemu_iothread_del($conf, $vmid, $deviceid);
4412
4413     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4414
4415         qemu_devicedel($vmid, $deviceid);
4416         qemu_devicedelverify($vmid, $deviceid);
4417         qemu_iothread_del($conf, $vmid, $deviceid);
4418
4419     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4420
4421         qemu_devicedel($vmid, $deviceid);
4422         qemu_drivedel($vmid, $deviceid);
4423         qemu_deletescsihw($conf, $vmid, $deviceid);
4424
4425     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4426
4427         qemu_devicedel($vmid, $deviceid);
4428         qemu_devicedelverify($vmid, $deviceid);
4429         qemu_netdevdel($vmid, $deviceid);
4430
4431     } else {
4432         die "can't unplug device '$deviceid'\n";
4433     }
4434
4435     return 1;
4436 }
4437
4438 sub qemu_deviceadd {
4439     my ($vmid, $devicefull) = @_;
4440
4441     $devicefull = "driver=".$devicefull;
4442     my %options =  split(/[=,]/, $devicefull);
4443
4444     vm_mon_cmd($vmid, "device_add" , %options);
4445 }
4446
4447 sub qemu_devicedel {
4448     my ($vmid, $deviceid) = @_;
4449
4450     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4451 }
4452
4453 sub qemu_iothread_add {
4454     my($vmid, $deviceid, $device) = @_;
4455
4456     if ($device->{iothread}) {
4457         my $iothreads = vm_iothreads_list($vmid);
4458         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4459     }
4460 }
4461
4462 sub qemu_iothread_del {
4463     my($conf, $vmid, $deviceid) = @_;
4464
4465     my $confid = $deviceid;
4466     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4467         $confid = 'scsi' . $1;
4468     }
4469     my $device = parse_drive($confid, $conf->{$confid});
4470     if ($device->{iothread}) {
4471         my $iothreads = vm_iothreads_list($vmid);
4472         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4473     }
4474 }
4475
4476 sub qemu_objectadd {
4477     my($vmid, $objectid, $qomtype) = @_;
4478
4479     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4480
4481     return 1;
4482 }
4483
4484 sub qemu_objectdel {
4485     my($vmid, $objectid) = @_;
4486
4487     vm_mon_cmd($vmid, "object-del", id => $objectid);
4488
4489     return 1;
4490 }
4491
4492 sub qemu_driveadd {
4493     my ($storecfg, $vmid, $device) = @_;
4494
4495     my $drive = print_drive_full($storecfg, $vmid, $device);
4496     $drive =~ s/\\/\\\\/g;
4497     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4498
4499     # If the command succeeds qemu prints: "OK"
4500     return 1 if $ret =~ m/OK/s;
4501
4502     die "adding drive failed: $ret\n";
4503 }
4504
4505 sub qemu_drivedel {
4506     my($vmid, $deviceid) = @_;
4507
4508     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4509     $ret =~ s/^\s+//;
4510
4511     return 1 if $ret eq "";
4512
4513     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4514     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4515
4516     die "deleting drive $deviceid failed : $ret\n";
4517 }
4518
4519 sub qemu_deviceaddverify {
4520     my ($vmid, $deviceid) = @_;
4521
4522     for (my $i = 0; $i <= 5; $i++) {
4523          my $devices_list = vm_devices_list($vmid);
4524          return 1 if defined($devices_list->{$deviceid});
4525          sleep 1;
4526     }
4527
4528     die "error on hotplug device '$deviceid'\n";
4529 }
4530
4531
4532 sub qemu_devicedelverify {
4533     my ($vmid, $deviceid) = @_;
4534
4535     # need to verify that the device is correctly removed as device_del
4536     # is async and empty return is not reliable
4537
4538     for (my $i = 0; $i <= 5; $i++) {
4539          my $devices_list = vm_devices_list($vmid);
4540          return 1 if !defined($devices_list->{$deviceid});
4541          sleep 1;
4542     }
4543
4544     die "error on hot-unplugging device '$deviceid'\n";
4545 }
4546
4547 sub qemu_findorcreatescsihw {
4548     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4549
4550     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4551
4552     my $scsihwid="$controller_prefix$controller";
4553     my $devices_list = vm_devices_list($vmid);
4554
4555     if(!defined($devices_list->{$scsihwid})) {
4556         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4557     }
4558
4559     return 1;
4560 }
4561
4562 sub qemu_deletescsihw {
4563     my ($conf, $vmid, $opt) = @_;
4564
4565     my $device = parse_drive($opt, $conf->{$opt});
4566
4567     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4568         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4569         return 1;
4570     }
4571
4572     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4573
4574     my $devices_list = vm_devices_list($vmid);
4575     foreach my $opt (keys %{$devices_list}) {
4576         if (PVE::QemuServer::is_valid_drivename($opt)) {
4577             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4578             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4579                 return 1;
4580             }
4581         }
4582     }
4583
4584     my $scsihwid="scsihw$controller";
4585
4586     vm_deviceunplug($vmid, $conf, $scsihwid);
4587
4588     return 1;
4589 }
4590
4591 sub qemu_add_pci_bridge {
4592     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4593
4594     my $bridges = {};
4595
4596     my $bridgeid;
4597
4598     print_pci_addr($device, $bridges, $arch, $machine_type);
4599
4600     while (my ($k, $v) = each %$bridges) {
4601         $bridgeid = $k;
4602     }
4603     return 1 if !defined($bridgeid) || $bridgeid < 1;
4604
4605     my $bridge = "pci.$bridgeid";
4606     my $devices_list = vm_devices_list($vmid);
4607
4608     if (!defined($devices_list->{$bridge})) {
4609         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4610     }
4611
4612     return 1;
4613 }
4614
4615 sub qemu_set_link_status {
4616     my ($vmid, $device, $up) = @_;
4617
4618     vm_mon_cmd($vmid, "set_link", name => $device,
4619                up => $up ? JSON::true : JSON::false);
4620 }
4621
4622 sub qemu_netdevadd {
4623     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4624
4625     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4626     my %options =  split(/[=,]/, $netdev);
4627
4628     vm_mon_cmd($vmid, "netdev_add",  %options);
4629     return 1;
4630 }
4631
4632 sub qemu_netdevdel {
4633     my ($vmid, $deviceid) = @_;
4634
4635     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4636 }
4637
4638 sub qemu_usb_hotplug {
4639     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4640
4641     return if !$device;
4642
4643     # remove the old one first
4644     vm_deviceunplug($vmid, $conf, $deviceid);
4645
4646     # check if xhci controller is necessary and available
4647     if ($device->{usb3}) {
4648
4649         my $devicelist = vm_devices_list($vmid);
4650
4651         if (!$devicelist->{xhci}) {
4652             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4653             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4654         }
4655     }
4656     my $d = parse_usb_device($device->{host});
4657     $d->{usb3} = $device->{usb3};
4658
4659     # add the new one
4660     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4661 }
4662
4663 sub qemu_cpu_hotplug {
4664     my ($vmid, $conf, $vcpus) = @_;
4665
4666     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4667
4668     my $sockets = 1;
4669     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4670     $sockets = $conf->{sockets} if  $conf->{sockets};
4671     my $cores = $conf->{cores} || 1;
4672     my $maxcpus = $sockets * $cores;
4673
4674     $vcpus = $maxcpus if !$vcpus;
4675
4676     die "you can't add more vcpus than maxcpus\n"
4677         if $vcpus > $maxcpus;
4678
4679     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4680
4681     if ($vcpus < $currentvcpus) {
4682
4683         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4684
4685             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4686                 qemu_devicedel($vmid, "cpu$i");
4687                 my $retry = 0;
4688                 my $currentrunningvcpus = undef;
4689                 while (1) {
4690                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4691                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4692                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4693                     $retry++;
4694                     sleep 1;
4695                 }
4696                 #update conf after each succesfull cpu unplug
4697                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4698                 PVE::QemuConfig->write_config($vmid, $conf);
4699             }
4700         } else {
4701             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4702         }
4703
4704         return;
4705     }
4706
4707     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4708     die "vcpus in running vm does not match its configuration\n"
4709         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4710
4711     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4712
4713         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4714             my $cpustr = print_cpu_device($conf, $i);
4715             qemu_deviceadd($vmid, $cpustr);
4716
4717             my $retry = 0;
4718             my $currentrunningvcpus = undef;
4719             while (1) {
4720                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4721                 last if scalar(@{$currentrunningvcpus}) == $i;
4722                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4723                 sleep 1;
4724                 $retry++;
4725             }
4726             #update conf after each succesfull cpu hotplug
4727             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4728             PVE::QemuConfig->write_config($vmid, $conf);
4729         }
4730     } else {
4731
4732         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4733             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4734         }
4735     }
4736 }
4737
4738 sub qemu_block_set_io_throttle {
4739     my ($vmid, $deviceid,
4740         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4741         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4742         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4743         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4744
4745     return if !check_running($vmid) ;
4746
4747     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4748         bps => int($bps),
4749         bps_rd => int($bps_rd),
4750         bps_wr => int($bps_wr),
4751         iops => int($iops),
4752         iops_rd => int($iops_rd),
4753         iops_wr => int($iops_wr),
4754         bps_max => int($bps_max),
4755         bps_rd_max => int($bps_rd_max),
4756         bps_wr_max => int($bps_wr_max),
4757         iops_max => int($iops_max),
4758         iops_rd_max => int($iops_rd_max),
4759         iops_wr_max => int($iops_wr_max),
4760         bps_max_length => int($bps_max_length),
4761         bps_rd_max_length => int($bps_rd_max_length),
4762         bps_wr_max_length => int($bps_wr_max_length),
4763         iops_max_length => int($iops_max_length),
4764         iops_rd_max_length => int($iops_rd_max_length),
4765         iops_wr_max_length => int($iops_wr_max_length),
4766     );
4767
4768 }
4769
4770 # old code, only used to shutdown old VM after update
4771 sub __read_avail {
4772     my ($fh, $timeout) = @_;
4773
4774     my $sel = new IO::Select;
4775     $sel->add($fh);
4776
4777     my $res = '';
4778     my $buf;
4779
4780     my @ready;
4781     while (scalar (@ready = $sel->can_read($timeout))) {
4782         my $count;
4783         if ($count = $fh->sysread($buf, 8192)) {
4784             if ($buf =~ /^(.*)\(qemu\) $/s) {
4785                 $res .= $1;
4786                 last;
4787             } else {
4788                 $res .= $buf;
4789             }
4790         } else {
4791             if (!defined($count)) {
4792                 die "$!\n";
4793             }
4794             last;
4795         }
4796     }
4797
4798     die "monitor read timeout\n" if !scalar(@ready);
4799
4800     return $res;
4801 }
4802
4803 sub qemu_block_resize {
4804     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4805
4806     my $running = check_running($vmid);
4807
4808     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4809
4810     return if !$running;
4811
4812     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4813
4814 }
4815
4816 sub qemu_volume_snapshot {
4817     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4818
4819     my $running = check_running($vmid);
4820
4821     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4822         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4823     } else {
4824         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4825     }
4826 }
4827
4828 sub qemu_volume_snapshot_delete {
4829     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4830
4831     my $running = check_running($vmid);
4832
4833     if($running) {
4834
4835         $running = undef;
4836         my $conf = PVE::QemuConfig->load_config($vmid);
4837         foreach_drive($conf, sub {
4838             my ($ds, $drive) = @_;
4839             $running = 1 if $drive->{file} eq $volid;
4840         });
4841     }
4842
4843     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4844         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4845     } else {
4846         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4847     }
4848 }
4849
4850 sub set_migration_caps {
4851     my ($vmid) = @_;
4852
4853     my $cap_ref = [];
4854
4855     my $enabled_cap = {
4856         "auto-converge" => 1,
4857         "xbzrle" => 1,
4858         "x-rdma-pin-all" => 0,
4859         "zero-blocks" => 0,
4860         "compress" => 0
4861     };
4862
4863     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4864
4865     for my $supported_capability (@$supported_capabilities) {
4866         push @$cap_ref, {
4867             capability => $supported_capability->{capability},
4868             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4869         };
4870     }
4871
4872     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4873 }
4874
4875 my $fast_plug_option = {
4876     'lock' => 1,
4877     'name' => 1,
4878     'onboot' => 1,
4879     'shares' => 1,
4880     'startup' => 1,
4881     'description' => 1,
4882     'protection' => 1,
4883     'vmstatestorage' => 1,
4884     'hookscript' => 1,
4885 };
4886
4887 # hotplug changes in [PENDING]
4888 # $selection hash can be used to only apply specified options, for
4889 # example: { cores => 1 } (only apply changed 'cores')
4890 # $errors ref is used to return error messages
4891 sub vmconfig_hotplug_pending {
4892     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4893
4894     my $defaults = load_defaults();
4895     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4896
4897     # commit values which do not have any impact on running VM first
4898     # Note: those option cannot raise errors, we we do not care about
4899     # $selection and always apply them.
4900
4901     my $add_error = sub {
4902         my ($opt, $msg) = @_;
4903         $errors->{$opt} = "hotplug problem - $msg";
4904     };
4905
4906     my $changes = 0;
4907     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4908         if ($fast_plug_option->{$opt}) {
4909             $conf->{$opt} = $conf->{pending}->{$opt};
4910             delete $conf->{pending}->{$opt};
4911             $changes = 1;
4912         }
4913     }
4914
4915     if ($changes) {
4916         PVE::QemuConfig->write_config($vmid, $conf);
4917         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4918     }
4919
4920     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4921
4922     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4923     while (my ($opt, $force) = each %$pending_delete_hash) {
4924         next if $selection && !$selection->{$opt};
4925         eval {
4926             if ($opt eq 'hotplug') {
4927                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4928             } elsif ($opt eq 'tablet') {
4929                 die "skip\n" if !$hotplug_features->{usb};
4930                 if ($defaults->{tablet}) {
4931                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4932                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4933                         if $arch eq 'aarch64';
4934                 } else {
4935                     vm_deviceunplug($vmid, $conf, 'tablet');
4936                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4937                 }
4938             } elsif ($opt =~ m/^usb\d+/) {
4939                 die "skip\n";
4940                 # since we cannot reliably hot unplug usb devices
4941                 # we are disabling it
4942                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4943                 vm_deviceunplug($vmid, $conf, $opt);
4944             } elsif ($opt eq 'vcpus') {
4945                 die "skip\n" if !$hotplug_features->{cpu};
4946                 qemu_cpu_hotplug($vmid, $conf, undef);
4947             } elsif ($opt eq 'balloon') {
4948                 # enable balloon device is not hotpluggable
4949                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4950                 # here we reset the ballooning value to memory
4951                 my $balloon = $conf->{memory} || $defaults->{memory};
4952                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4953             } elsif ($fast_plug_option->{$opt}) {
4954                 # do nothing
4955             } elsif ($opt =~ m/^net(\d+)$/) {
4956                 die "skip\n" if !$hotplug_features->{network};
4957                 vm_deviceunplug($vmid, $conf, $opt);
4958             } elsif (is_valid_drivename($opt)) {
4959                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4960                 vm_deviceunplug($vmid, $conf, $opt);
4961                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4962             } elsif ($opt =~ m/^memory$/) {
4963                 die "skip\n" if !$hotplug_features->{memory};
4964                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4965             } elsif ($opt eq 'cpuunits') {
4966                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4967             } elsif ($opt eq 'cpulimit') {
4968                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4969             } else {
4970                 die "skip\n";
4971             }
4972         };
4973         if (my $err = $@) {
4974             &$add_error($opt, $err) if $err ne "skip\n";
4975         } else {
4976             # save new config if hotplug was successful
4977             delete $conf->{$opt};
4978             vmconfig_undelete_pending_option($conf, $opt);
4979             PVE::QemuConfig->write_config($vmid, $conf);
4980             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4981         }
4982     }
4983
4984     my $apply_pending_cloudinit;
4985     $apply_pending_cloudinit = sub {
4986         my ($key, $value) = @_;
4987         $apply_pending_cloudinit = sub {}; # once is enough
4988
4989         my @cloudinit_opts = keys %$confdesc_cloudinit;
4990         foreach my $opt (keys %{$conf->{pending}}) {
4991             next if !grep { $_ eq $opt } @cloudinit_opts;
4992             $conf->{$opt} = delete $conf->{pending}->{$opt};
4993         }
4994
4995         my $new_conf = { %$conf };
4996         $new_conf->{$key} = $value;
4997         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4998     };
4999
5000     foreach my $opt (keys %{$conf->{pending}}) {
5001         next if $selection && !$selection->{$opt};
5002         my $value = $conf->{pending}->{$opt};
5003         eval {
5004             if ($opt eq 'hotplug') {
5005                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5006             } elsif ($opt eq 'tablet') {
5007                 die "skip\n" if !$hotplug_features->{usb};
5008                 if ($value == 1) {
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5010                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5011                         if $arch eq 'aarch64';
5012                 } elsif ($value == 0) {
5013                     vm_deviceunplug($vmid, $conf, 'tablet');
5014                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5015                 }
5016             } elsif ($opt =~ m/^usb\d+$/) {
5017                 die "skip\n";
5018                 # since we cannot reliably hot unplug usb devices
5019                 # we are disabling it
5020                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5021                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5022                 die "skip\n" if !$d;
5023                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5024             } elsif ($opt eq 'vcpus') {
5025                 die "skip\n" if !$hotplug_features->{cpu};
5026                 qemu_cpu_hotplug($vmid, $conf, $value);
5027             } elsif ($opt eq 'balloon') {
5028                 # enable/disable balloning device is not hotpluggable
5029                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5030                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5031                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5032
5033                 # allow manual ballooning if shares is set to zero
5034                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5035                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5036                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5037                 }
5038             } elsif ($opt =~ m/^net(\d+)$/) {
5039                 # some changes can be done without hotplug
5040                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5041                                     $vmid, $opt, $value, $arch, $machine_type);
5042             } elsif (is_valid_drivename($opt)) {
5043                 # some changes can be done without hotplug
5044                 my $drive = parse_drive($opt, $value);
5045                 if (drive_is_cloudinit($drive)) {
5046                     &$apply_pending_cloudinit($opt, $value);
5047                 }
5048                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5049                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5050             } elsif ($opt =~ m/^memory$/) { #dimms
5051                 die "skip\n" if !$hotplug_features->{memory};
5052                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5053             } elsif ($opt eq 'cpuunits') {
5054                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5055             } elsif ($opt eq 'cpulimit') {
5056                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5057                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5058             } else {
5059                 die "skip\n";  # skip non-hot-pluggable options
5060             }
5061         };
5062         if (my $err = $@) {
5063             &$add_error($opt, $err) if $err ne "skip\n";
5064         } else {
5065             # save new config if hotplug was successful
5066             $conf->{$opt} = $value;
5067             delete $conf->{pending}->{$opt};
5068             PVE::QemuConfig->write_config($vmid, $conf);
5069             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5070         }
5071     }
5072 }
5073
5074 sub try_deallocate_drive {
5075     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5076
5077     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5078         my $volid = $drive->{file};
5079         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5080             my $sid = PVE::Storage::parse_volume_id($volid);
5081             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5082
5083             # check if the disk is really unused
5084             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5085                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5086             PVE::Storage::vdisk_free($storecfg, $volid);
5087             return 1;
5088         } else {
5089             # If vm is not owner of this disk remove from config
5090             return 1;
5091         }
5092     }
5093
5094     return undef;
5095 }
5096
5097 sub vmconfig_delete_or_detach_drive {
5098     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5099
5100     my $drive = parse_drive($opt, $conf->{$opt});
5101
5102     my $rpcenv = PVE::RPCEnvironment::get();
5103     my $authuser = $rpcenv->get_user();
5104
5105     if ($force) {
5106         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5107         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5108     } else {
5109         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5110     }
5111 }
5112
5113 sub vmconfig_apply_pending {
5114     my ($vmid, $conf, $storecfg) = @_;
5115
5116     # cold plug
5117
5118     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5119     while (my ($opt, $force) = each %$pending_delete_hash) {
5120         die "internal error" if $opt =~ m/^unused/;
5121         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5122         if (!defined($conf->{$opt})) {
5123             vmconfig_undelete_pending_option($conf, $opt);
5124             PVE::QemuConfig->write_config($vmid, $conf);
5125         } elsif (is_valid_drivename($opt)) {
5126             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5127             vmconfig_undelete_pending_option($conf, $opt);
5128             delete $conf->{$opt};
5129             PVE::QemuConfig->write_config($vmid, $conf);
5130         } else {
5131             vmconfig_undelete_pending_option($conf, $opt);
5132             delete $conf->{$opt};
5133             PVE::QemuConfig->write_config($vmid, $conf);
5134         }
5135     }
5136
5137     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5138
5139     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5140         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5141
5142         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5143             # skip if nothing changed
5144         } elsif (is_valid_drivename($opt)) {
5145             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5146                 if defined($conf->{$opt});
5147             $conf->{$opt} = $conf->{pending}->{$opt};
5148         } else {
5149             $conf->{$opt} = $conf->{pending}->{$opt};
5150         }
5151
5152         delete $conf->{pending}->{$opt};
5153         PVE::QemuConfig->write_config($vmid, $conf);
5154     }
5155 }
5156
5157 my $safe_num_ne = sub {
5158     my ($a, $b) = @_;
5159
5160     return 0 if !defined($a) && !defined($b);
5161     return 1 if !defined($a);
5162     return 1 if !defined($b);
5163
5164     return $a != $b;
5165 };
5166
5167 my $safe_string_ne = sub {
5168     my ($a, $b) = @_;
5169
5170     return 0 if !defined($a) && !defined($b);
5171     return 1 if !defined($a);
5172     return 1 if !defined($b);
5173
5174     return $a ne $b;
5175 };
5176
5177 sub vmconfig_update_net {
5178     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5179
5180     my $newnet = parse_net($value);
5181
5182     if ($conf->{$opt}) {
5183         my $oldnet = parse_net($conf->{$opt});
5184
5185         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5186             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5187             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5188             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5189
5190             # for non online change, we try to hot-unplug
5191             die "skip\n" if !$hotplug;
5192             vm_deviceunplug($vmid, $conf, $opt);
5193         } else {
5194
5195             die "internal error" if $opt !~ m/net(\d+)/;
5196             my $iface = "tap${vmid}i$1";
5197
5198             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5199                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5200                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5201                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5202                 PVE::Network::tap_unplug($iface);
5203                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5204             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5205                 # Rate can be applied on its own but any change above needs to
5206                 # include the rate in tap_plug since OVS resets everything.
5207                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5208             }
5209
5210             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5211                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5212             }
5213
5214             return 1;
5215         }
5216     }
5217
5218     if ($hotplug) {
5219         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5220     } else {
5221         die "skip\n";
5222     }
5223 }
5224
5225 sub vmconfig_update_disk {
5226     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5227
5228     # fixme: do we need force?
5229
5230     my $drive = parse_drive($opt, $value);
5231
5232     if ($conf->{$opt}) {
5233
5234         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5235
5236             my $media = $drive->{media} || 'disk';
5237             my $oldmedia = $old_drive->{media} || 'disk';
5238             die "unable to change media type\n" if $media ne $oldmedia;
5239
5240             if (!drive_is_cdrom($old_drive)) {
5241
5242                 if ($drive->{file} ne $old_drive->{file}) {
5243
5244                     die "skip\n" if !$hotplug;
5245
5246                     # unplug and register as unused
5247                     vm_deviceunplug($vmid, $conf, $opt);
5248                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5249
5250                 } else {
5251                     # update existing disk
5252
5253                     # skip non hotpluggable value
5254                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5255                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5256                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5257                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5258                         die "skip\n";
5259                     }
5260
5261                     # apply throttle
5262                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5263                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5264                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5265                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5266                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5267                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5268                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5269                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5270                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5271                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5272                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5273                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5274                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5275                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5276                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5277                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5278                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5279                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5280
5281                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5282                                                    ($drive->{mbps} || 0)*1024*1024,
5283                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5284                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5285                                                    $drive->{iops} || 0,
5286                                                    $drive->{iops_rd} || 0,
5287                                                    $drive->{iops_wr} || 0,
5288                                                    ($drive->{mbps_max} || 0)*1024*1024,
5289                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5290                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5291                                                    $drive->{iops_max} || 0,
5292                                                    $drive->{iops_rd_max} || 0,
5293                                                    $drive->{iops_wr_max} || 0,
5294                                                    $drive->{bps_max_length} || 1,
5295                                                    $drive->{bps_rd_max_length} || 1,
5296                                                    $drive->{bps_wr_max_length} || 1,
5297                                                    $drive->{iops_max_length} || 1,
5298                                                    $drive->{iops_rd_max_length} || 1,
5299                                                    $drive->{iops_wr_max_length} || 1);
5300
5301                     }
5302
5303                     return 1;
5304                 }
5305
5306             } else { # cdrom
5307
5308                 if ($drive->{file} eq 'none') {
5309                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5310                     if (drive_is_cloudinit($old_drive)) {
5311                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5312                     }
5313                 } else {
5314                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5315                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5316                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5317                 }
5318
5319                 return 1;
5320             }
5321         }
5322     }
5323
5324     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5325     # hotplug new disks
5326     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5327     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5328 }
5329
5330 sub vm_start {
5331     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5332         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5333
5334     PVE::QemuConfig->lock_config($vmid, sub {
5335         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5336
5337         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5338
5339         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5340
5341         PVE::QemuConfig->check_lock($conf)
5342             if !($skiplock || $is_suspended);
5343
5344         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5345
5346         # clean up leftover reboot request files
5347         eval { clear_reboot_request($vmid); };
5348         warn $@ if $@;
5349
5350         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5351             vmconfig_apply_pending($vmid, $conf, $storecfg);
5352             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5353         }
5354
5355         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5356
5357         my $defaults = load_defaults();
5358
5359         # set environment variable useful inside network script
5360         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5361
5362         my $local_volumes = {};
5363
5364         if ($targetstorage) {
5365             foreach_drive($conf, sub {
5366                 my ($ds, $drive) = @_;
5367
5368                 return if drive_is_cdrom($drive);
5369
5370                 my $volid = $drive->{file};
5371
5372                 return if !$volid;
5373
5374                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5375
5376                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5377                 return if $scfg->{shared};
5378                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5379             });
5380
5381             my $format = undef;
5382
5383             foreach my $opt (sort keys %$local_volumes) {
5384
5385                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5386                 my $drive = parse_drive($opt, $conf->{$opt});
5387
5388                 #if remote storage is specified, use default format
5389                 if ($targetstorage && $targetstorage ne "1") {
5390                     $storeid = $targetstorage;
5391                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5392                     $format = $defFormat;
5393                 } else {
5394                     #else we use same format than original
5395                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5396                     $format = qemu_img_format($scfg, $volid);
5397                 }
5398
5399                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5400                 my $newdrive = $drive;
5401                 $newdrive->{format} = $format;
5402                 $newdrive->{file} = $newvolid;
5403                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5404                 $local_volumes->{$opt} = $drivestr;
5405                 #pass drive to conf for command line
5406                 $conf->{$opt} = $drivestr;
5407             }
5408         }
5409
5410         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5411
5412         if ($is_suspended) {
5413             # enforce machine type on suspended vm to ensure HW compatibility
5414             $forcemachine = $conf->{runningmachine};
5415             print "Resuming suspended VM\n";
5416         }
5417
5418         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5419
5420         my $migrate_port = 0;
5421         my $migrate_uri;
5422         if ($statefile) {
5423             if ($statefile eq 'tcp') {
5424                 my $localip = "localhost";
5425                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5426                 my $nodename = PVE::INotify::nodename();
5427
5428                 if (!defined($migration_type)) {
5429                     if (defined($datacenterconf->{migration}->{type})) {
5430                         $migration_type = $datacenterconf->{migration}->{type};
5431                     } else {
5432                         $migration_type = 'secure';
5433                     }
5434                 }
5435
5436                 if ($migration_type eq 'insecure') {
5437                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5438                     if ($migrate_network_addr) {
5439                         $localip = $migrate_network_addr;
5440                     } else {
5441                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5442                     }
5443
5444                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5445                 }
5446
5447                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5448                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5449                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5450                 push @$cmd, '-incoming', $migrate_uri;
5451                 push @$cmd, '-S';
5452
5453             } elsif ($statefile eq 'unix') {
5454                 # should be default for secure migrations as a ssh TCP forward
5455                 # tunnel is not deterministic reliable ready and fails regurarly
5456                 # to set up in time, so use UNIX socket forwards
5457                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5458                 unlink $socket_addr;
5459
5460                 $migrate_uri = "unix:$socket_addr";
5461
5462                 push @$cmd, '-incoming', $migrate_uri;
5463                 push @$cmd, '-S';
5464
5465             } else {
5466                 push @$cmd, '-loadstate', $statefile;
5467             }
5468         } elsif ($paused) {
5469             push @$cmd, '-S';
5470         }
5471
5472         # host pci devices
5473         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5474           my $d = parse_hostpci($conf->{"hostpci$i"});
5475           next if !$d;
5476           my $pcidevices = $d->{pciid};
5477           foreach my $pcidevice (@$pcidevices) {
5478                 my $pciid = $pcidevice->{id};
5479
5480                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5481                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5482                 die "no pci device info for device '$pciid'\n" if !$info;
5483
5484                 if ($d->{mdev}) {
5485                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5486                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5487                 } else {
5488                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5489                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5490                     die "can't reset pci device '$pciid'\n"
5491                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5492                 }
5493           }
5494         }
5495
5496         PVE::Storage::activate_volumes($storecfg, $vollist);
5497
5498         eval {
5499             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5500                 outfunc => sub {}, errfunc => sub {});
5501         };
5502         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5503         # timeout should be more than enough here...
5504         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5505
5506         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5507                                                   : $defaults->{cpuunits};
5508
5509         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5510         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5511
5512         my %properties = (
5513             Slice => 'qemu.slice',
5514             KillMode => 'none',
5515             CPUShares => $cpuunits
5516         );
5517
5518         if (my $cpulimit = $conf->{cpulimit}) {
5519             $properties{CPUQuota} = int($cpulimit * 100);
5520         }
5521         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5522
5523         my $run_qemu = sub {
5524             PVE::Tools::run_fork sub {
5525                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5526                 run_command($cmd, %run_params);
5527             };
5528         };
5529
5530         if ($conf->{hugepages}) {
5531
5532             my $code = sub {
5533                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5534                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5535
5536                 PVE::QemuServer::Memory::hugepages_mount();
5537                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5538
5539                 eval { $run_qemu->() };
5540                 if (my $err = $@) {
5541                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5542                     die $err;
5543                 }
5544
5545                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5546             };
5547             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5548
5549         } else {
5550             eval { $run_qemu->() };
5551         }
5552
5553         if (my $err = $@) {
5554             # deactivate volumes if start fails
5555             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5556             die "start failed: $err";
5557         }
5558
5559         print "migration listens on $migrate_uri\n" if $migrate_uri;
5560
5561         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5562             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5563             warn $@ if $@;
5564         }
5565
5566         #start nbd server for storage migration
5567         if ($targetstorage) {
5568             my $nodename = PVE::INotify::nodename();
5569             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5570             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5571             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5572             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5573
5574             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5575
5576             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5577
5578             foreach my $opt (sort keys %$local_volumes) {
5579                 my $volid = $local_volumes->{$opt};
5580                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5581                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5582                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5583             }
5584         }
5585
5586         if ($migratedfrom) {
5587             eval {
5588                 set_migration_caps($vmid);
5589             };
5590             warn $@ if $@;
5591
5592             if ($spice_port) {
5593                 print "spice listens on port $spice_port\n";
5594                 if ($spice_ticket) {
5595                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5596                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5597                 }
5598             }
5599
5600         } else {
5601             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5602                 if !$statefile && $conf->{balloon};
5603
5604             foreach my $opt (keys %$conf) {
5605                 next if $opt !~  m/^net\d+$/;
5606                 my $nicconf = parse_net($conf->{$opt});
5607                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5608             }
5609         }
5610
5611         vm_mon_cmd_nocheck($vmid, 'qom-set',
5612                     path => "machine/peripheral/balloon0",
5613                     property => "guest-stats-polling-interval",
5614                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5615
5616         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5617             print "Resumed VM, removing state\n";
5618             delete $conf->@{qw(lock vmstate runningmachine)};
5619             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5620             PVE::Storage::vdisk_free($storecfg, $vmstate);
5621             PVE::QemuConfig->write_config($vmid, $conf);
5622         }
5623
5624         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5625     });
5626 }
5627
5628 sub vm_mon_cmd {
5629     my ($vmid, $execute, %params) = @_;
5630
5631     my $cmd = { execute => $execute, arguments => \%params };
5632     vm_qmp_command($vmid, $cmd);
5633 }
5634
5635 sub vm_mon_cmd_nocheck {
5636     my ($vmid, $execute, %params) = @_;
5637
5638     my $cmd = { execute => $execute, arguments => \%params };
5639     vm_qmp_command($vmid, $cmd, 1);
5640 }
5641
5642 sub vm_qmp_command {
5643     my ($vmid, $cmd, $nocheck) = @_;
5644
5645     my $res;
5646
5647     my $timeout;
5648     if ($cmd->{arguments}) {
5649         $timeout = delete $cmd->{arguments}->{timeout};
5650     }
5651
5652     eval {
5653         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5654         my $sname = qmp_socket($vmid);
5655         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5656             my $qmpclient = PVE::QMPClient->new();
5657
5658             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5659         } else {
5660             die "unable to open monitor socket\n";
5661         }
5662     };
5663     if (my $err = $@) {
5664         syslog("err", "VM $vmid qmp command failed - $err");
5665         die $err;
5666     }
5667
5668     return $res;
5669 }
5670
5671 sub vm_human_monitor_command {
5672     my ($vmid, $cmdline) = @_;
5673
5674     my $cmd = {
5675         execute => 'human-monitor-command',
5676         arguments => { 'command-line' => $cmdline},
5677     };
5678
5679     return vm_qmp_command($vmid, $cmd);
5680 }
5681
5682 sub vm_commandline {
5683     my ($storecfg, $vmid, $snapname) = @_;
5684
5685     my $conf = PVE::QemuConfig->load_config($vmid);
5686
5687     if ($snapname) {
5688         my $snapshot = $conf->{snapshots}->{$snapname};
5689         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5690
5691         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5692
5693         $conf = $snapshot;
5694     }
5695
5696     my $defaults = load_defaults();
5697
5698     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5699
5700     return PVE::Tools::cmd2string($cmd);
5701 }
5702
5703 sub vm_reset {
5704     my ($vmid, $skiplock) = @_;
5705
5706     PVE::QemuConfig->lock_config($vmid, sub {
5707
5708         my $conf = PVE::QemuConfig->load_config($vmid);
5709
5710         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5711
5712         vm_mon_cmd($vmid, "system_reset");
5713     });
5714 }
5715
5716 sub get_vm_volumes {
5717     my ($conf) = @_;
5718
5719     my $vollist = [];
5720     foreach_volid($conf, sub {
5721         my ($volid, $attr) = @_;
5722
5723         return if $volid =~ m|^/|;
5724
5725         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5726         return if !$sid;
5727
5728         push @$vollist, $volid;
5729     });
5730
5731     return $vollist;
5732 }
5733
5734 sub vm_stop_cleanup {
5735     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5736
5737     eval {
5738
5739         if (!$keepActive) {
5740             my $vollist = get_vm_volumes($conf);
5741             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5742         }
5743
5744         foreach my $ext (qw(mon qmp pid vnc qga)) {
5745             unlink "/var/run/qemu-server/${vmid}.$ext";
5746         }
5747
5748         if ($conf->{ivshmem}) {
5749             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5750             # just delete it for now, VMs which have this already open do not
5751             # are affected, but new VMs will get a separated one. If this
5752             # becomes an issue we either add some sort of ref-counting or just
5753             # add a "don't delete on stop" flag to the ivshmem format.
5754             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5755         }
5756
5757         foreach my $key (keys %$conf) {
5758             next if $key !~ m/^hostpci(\d+)$/;
5759             my $hostpciindex = $1;
5760             my $d = parse_hostpci($conf->{$key});
5761             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5762
5763             foreach my $pci (@{$d->{pciid}}) {
5764                 my $pciid = $pci->{id};
5765                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5766             }
5767         }
5768
5769         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5770     };
5771     warn $@ if $@; # avoid errors - just warn
5772 }
5773
5774 # call only in locked context
5775 sub _do_vm_stop {
5776     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5777
5778     my $pid = check_running($vmid, $nocheck);
5779     return if !$pid;
5780
5781     my $conf;
5782     if (!$nocheck) {
5783         $conf = PVE::QemuConfig->load_config($vmid);
5784         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5785         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5786             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5787             $timeout = $opts->{down} if $opts->{down};
5788         }
5789         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5790     }
5791
5792     eval {
5793         if ($shutdown) {
5794             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5795                 vm_qmp_command($vmid, {
5796                         execute => "guest-shutdown",
5797                         arguments => { timeout => $timeout }
5798                     }, $nocheck);
5799             } else {
5800                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5801             }
5802         } else {
5803             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5804         }
5805     };
5806     my $err = $@;
5807
5808     if (!$err) {
5809         $timeout = 60 if !defined($timeout);
5810
5811         my $count = 0;
5812         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5813             $count++;
5814             sleep 1;
5815         }
5816
5817         if ($count >= $timeout) {
5818             if ($force) {
5819                 warn "VM still running - terminating now with SIGTERM\n";
5820                 kill 15, $pid;
5821             } else {
5822                 die "VM quit/powerdown failed - got timeout\n";
5823             }
5824         } else {
5825             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5826             return;
5827         }
5828     } else {
5829         if ($force) {
5830             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5831             kill 15, $pid;
5832         } else {
5833             die "VM quit/powerdown failed\n";
5834         }
5835     }
5836
5837     # wait again
5838     $timeout = 10;
5839
5840     my $count = 0;
5841     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5842         $count++;
5843         sleep 1;
5844     }
5845
5846     if ($count >= $timeout) {
5847         warn "VM still running - terminating now with SIGKILL\n";
5848         kill 9, $pid;
5849         sleep 1;
5850     }
5851
5852     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5853 }
5854
5855 # Note: use $nocheck to skip tests if VM configuration file exists.
5856 # We need that when migration VMs to other nodes (files already moved)
5857 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5858 sub vm_stop {
5859     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5860
5861     $force = 1 if !defined($force) && !$shutdown;
5862
5863     if ($migratedfrom){
5864         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5865         kill 15, $pid if $pid;
5866         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5867         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5868         return;
5869     }
5870
5871     PVE::QemuConfig->lock_config($vmid, sub {
5872         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5873    });
5874 }
5875
5876 sub vm_reboot {
5877     my ($vmid, $timeout) = @_;
5878
5879     PVE::QemuConfig->lock_config($vmid, sub {
5880
5881         # only reboot if running, as qmeventd starts it again on a stop event
5882         return if !check_running($vmid);
5883
5884         create_reboot_request($vmid);
5885
5886         my $storecfg = PVE::Storage::config();
5887         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5888
5889    });
5890 }
5891
5892 sub vm_suspend {
5893     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5894
5895     my $conf;
5896     my $path;
5897     my $storecfg;
5898     my $vmstate;
5899
5900     PVE::QemuConfig->lock_config($vmid, sub {
5901
5902         $conf = PVE::QemuConfig->load_config($vmid);
5903
5904         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5905         PVE::QemuConfig->check_lock($conf)
5906             if !($skiplock || $is_backing_up);
5907
5908         die "cannot suspend to disk during backup\n"
5909             if $is_backing_up && $includestate;
5910
5911         if ($includestate) {
5912             $conf->{lock} = 'suspending';
5913             my $date = strftime("%Y-%m-%d", localtime(time()));
5914             $storecfg = PVE::Storage::config();
5915             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5916             $path = PVE::Storage::path($storecfg, $vmstate);
5917             PVE::QemuConfig->write_config($vmid, $conf);
5918         } else {
5919             vm_mon_cmd($vmid, "stop");
5920         }
5921     });
5922
5923     if ($includestate) {
5924         # save vm state
5925         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5926
5927         eval {
5928             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5929             for(;;) {
5930                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5931                 if (!$state->{status}) {
5932                     die "savevm not active\n";
5933                 } elsif ($state->{status} eq 'active') {
5934                     sleep(1);
5935                     next;
5936                 } elsif ($state->{status} eq 'completed') {
5937                     print "State saved, quitting\n";
5938                     last;
5939                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5940                     die "query-savevm failed with error '$state->{error}'\n"
5941                 } else {
5942                     die "query-savevm returned status '$state->{status}'\n";
5943                 }
5944             }
5945         };
5946         my $err = $@;
5947
5948         PVE::QemuConfig->lock_config($vmid, sub {
5949             $conf = PVE::QemuConfig->load_config($vmid);
5950             if ($err) {
5951                 # cleanup, but leave suspending lock, to indicate something went wrong
5952                 eval {
5953                     vm_mon_cmd($vmid, "savevm-end");
5954                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5955                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5956                     delete $conf->@{qw(vmstate runningmachine)};
5957                     PVE::QemuConfig->write_config($vmid, $conf);
5958                 };
5959                 warn $@ if $@;
5960                 die $err;
5961             }
5962
5963             die "lock changed unexpectedly\n"
5964                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5965
5966             vm_qmp_command($vmid, { execute => "quit" });
5967             $conf->{lock} = 'suspended';
5968             PVE::QemuConfig->write_config($vmid, $conf);
5969         });
5970     }
5971 }
5972
5973 sub vm_resume {
5974     my ($vmid, $skiplock, $nocheck) = @_;
5975
5976     PVE::QemuConfig->lock_config($vmid, sub {
5977         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5978         my $res = $vm_mon_cmd->($vmid, 'query-status');
5979         my $resume_cmd = 'cont';
5980
5981         if ($res->{status} && $res->{status} eq 'suspended') {
5982             $resume_cmd = 'system_wakeup';
5983         }
5984
5985         if (!$nocheck) {
5986
5987             my $conf = PVE::QemuConfig->load_config($vmid);
5988
5989             PVE::QemuConfig->check_lock($conf)
5990                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5991         }
5992
5993         $vm_mon_cmd->($vmid, $resume_cmd);
5994     });
5995 }
5996
5997 sub vm_sendkey {
5998     my ($vmid, $skiplock, $key) = @_;
5999
6000     PVE::QemuConfig->lock_config($vmid, sub {
6001
6002         my $conf = PVE::QemuConfig->load_config($vmid);
6003
6004         # there is no qmp command, so we use the human monitor command
6005         my $res = vm_human_monitor_command($vmid, "sendkey $key");
6006         die $res if $res ne '';
6007     });
6008 }
6009
6010 sub vm_destroy {
6011     my ($storecfg, $vmid, $skiplock) = @_;
6012
6013     PVE::QemuConfig->lock_config($vmid, sub {
6014
6015         my $conf = PVE::QemuConfig->load_config($vmid);
6016
6017         if (!check_running($vmid)) {
6018             destroy_vm($storecfg, $vmid, undef, $skiplock);
6019         } else {
6020             die "VM $vmid is running - destroy failed\n";
6021         }
6022     });
6023 }
6024
6025 # vzdump restore implementaion
6026
6027 sub tar_archive_read_firstfile {
6028     my $archive = shift;
6029
6030     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6031
6032     # try to detect archive type first
6033     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6034         die "unable to open file '$archive'\n";
6035     my $firstfile = <$fh>;
6036     kill 15, $pid;
6037     close $fh;
6038
6039     die "ERROR: archive contaions no data\n" if !$firstfile;
6040     chomp $firstfile;
6041
6042     return $firstfile;
6043 }
6044
6045 sub tar_restore_cleanup {
6046     my ($storecfg, $statfile) = @_;
6047
6048     print STDERR "starting cleanup\n";
6049
6050     if (my $fd = IO::File->new($statfile, "r")) {
6051         while (defined(my $line = <$fd>)) {
6052             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6053                 my $volid = $2;
6054                 eval {
6055                     if ($volid =~ m|^/|) {
6056                         unlink $volid || die 'unlink failed\n';
6057                     } else {
6058                         PVE::Storage::vdisk_free($storecfg, $volid);
6059                     }
6060                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6061                 };
6062                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6063             } else {
6064                 print STDERR "unable to parse line in statfile - $line";
6065             }
6066         }
6067         $fd->close();
6068     }
6069 }
6070
6071 sub restore_archive {
6072     my ($archive, $vmid, $user, $opts) = @_;
6073
6074     my $format = $opts->{format};
6075     my $comp;
6076
6077     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6078         $format = 'tar' if !$format;
6079         $comp = 'gzip';
6080     } elsif ($archive =~ m/\.tar$/) {
6081         $format = 'tar' if !$format;
6082     } elsif ($archive =~ m/.tar.lzo$/) {
6083         $format = 'tar' if !$format;
6084         $comp = 'lzop';
6085     } elsif ($archive =~ m/\.vma$/) {
6086         $format = 'vma' if !$format;
6087     } elsif ($archive =~ m/\.vma\.gz$/) {
6088         $format = 'vma' if !$format;
6089         $comp = 'gzip';
6090     } elsif ($archive =~ m/\.vma\.lzo$/) {
6091         $format = 'vma' if !$format;
6092         $comp = 'lzop';
6093     } else {
6094         $format = 'vma' if !$format; # default
6095     }
6096
6097     # try to detect archive format
6098     if ($format eq 'tar') {
6099         return restore_tar_archive($archive, $vmid, $user, $opts);
6100     } else {
6101         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6102     }
6103 }
6104
6105 sub restore_update_config_line {
6106     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6107
6108     return if $line =~ m/^\#qmdump\#/;
6109     return if $line =~ m/^\#vzdump\#/;
6110     return if $line =~ m/^lock:/;
6111     return if $line =~ m/^unused\d+:/;
6112     return if $line =~ m/^parent:/;
6113
6114     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6115     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6116         # try to convert old 1.X settings
6117         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6118         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6119             my ($model, $macaddr) = split(/\=/, $devconfig);
6120             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6121             my $net = {
6122                 model => $model,
6123                 bridge => "vmbr$ind",
6124                 macaddr => $macaddr,
6125             };
6126             my $netstr = print_net($net);
6127
6128             print $outfd "net$cookie->{netcount}: $netstr\n";
6129             $cookie->{netcount}++;
6130         }
6131     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6132         my ($id, $netstr) = ($1, $2);
6133         my $net = parse_net($netstr);
6134         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6135         $netstr = print_net($net);
6136         print $outfd "$id: $netstr\n";
6137     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6138         my $virtdev = $1;
6139         my $value = $3;
6140         my $di = parse_drive($virtdev, $value);
6141         if (defined($di->{backup}) && !$di->{backup}) {
6142             print $outfd "#$line";
6143         } elsif ($map->{$virtdev}) {
6144             delete $di->{format}; # format can change on restore
6145             $di->{file} = $map->{$virtdev};
6146             $value = print_drive($vmid, $di);
6147             print $outfd "$virtdev: $value\n";
6148         } else {
6149             print $outfd $line;
6150         }
6151     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6152         my $vmgenid = $1;
6153         if ($vmgenid ne '0') {
6154             # always generate a new vmgenid if there was a valid one setup
6155             $vmgenid = generate_uuid();
6156         }
6157         print $outfd "vmgenid: $vmgenid\n";
6158     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6159         my ($uuid, $uuid_str);
6160         UUID::generate($uuid);
6161         UUID::unparse($uuid, $uuid_str);
6162         my $smbios1 = parse_smbios1($2);
6163         $smbios1->{uuid} = $uuid_str;
6164         print $outfd $1.print_smbios1($smbios1)."\n";
6165     } else {
6166         print $outfd $line;
6167     }
6168 }
6169
6170 sub scan_volids {
6171     my ($cfg, $vmid) = @_;
6172
6173     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6174
6175     my $volid_hash = {};
6176     foreach my $storeid (keys %$info) {
6177         foreach my $item (@{$info->{$storeid}}) {
6178             next if !($item->{volid} && $item->{size});
6179             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6180             $volid_hash->{$item->{volid}} = $item;
6181         }
6182     }
6183
6184     return $volid_hash;
6185 }
6186
6187 sub is_volume_in_use {
6188     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6189
6190     my $path = PVE::Storage::path($storecfg, $volid);
6191
6192     my $scan_config = sub {
6193         my ($cref, $snapname) = @_;
6194
6195         foreach my $key (keys %$cref) {
6196             my $value = $cref->{$key};
6197             if (is_valid_drivename($key)) {
6198                 next if $skip_drive && $key eq $skip_drive;
6199                 my $drive = parse_drive($key, $value);
6200                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6201                 return 1 if $volid eq $drive->{file};
6202                 if ($drive->{file} =~ m!^/!) {
6203                     return 1 if $drive->{file} eq $path;
6204                 } else {
6205                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6206                     next if !$storeid;
6207                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6208                     next if !$scfg;
6209                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6210                 }
6211             }
6212         }
6213
6214         return 0;
6215     };
6216
6217     return 1 if &$scan_config($conf);
6218
6219     undef $skip_drive;
6220
6221     foreach my $snapname (keys %{$conf->{snapshots}}) {
6222         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6223     }
6224
6225     return 0;
6226 }
6227
6228 sub update_disksize {
6229     my ($vmid, $conf, $volid_hash) = @_;
6230
6231     my $changes;
6232     my $prefix = "VM $vmid:";
6233
6234     # used and unused disks
6235     my $referenced = {};
6236
6237     # Note: it is allowed to define multiple storages with same path (alias), so
6238     # we need to check both 'volid' and real 'path' (two different volid can point
6239     # to the same path).
6240
6241     my $referencedpath = {};
6242
6243     # update size info
6244     foreach my $opt (keys %$conf) {
6245         if (is_valid_drivename($opt)) {
6246             my $drive = parse_drive($opt, $conf->{$opt});
6247             my $volid = $drive->{file};
6248             next if !$volid;
6249
6250             $referenced->{$volid} = 1;
6251             if ($volid_hash->{$volid} &&
6252                 (my $path = $volid_hash->{$volid}->{path})) {
6253                 $referencedpath->{$path} = 1;
6254             }
6255
6256             next if drive_is_cdrom($drive);
6257             next if !$volid_hash->{$volid};
6258
6259             $drive->{size} = $volid_hash->{$volid}->{size};
6260             my $new = print_drive($vmid, $drive);
6261             if ($new ne $conf->{$opt}) {
6262                 $changes = 1;
6263                 $conf->{$opt} = $new;
6264                 print "$prefix update disk '$opt' information.\n";
6265             }
6266         }
6267     }
6268
6269     # remove 'unusedX' entry if volume is used
6270     foreach my $opt (keys %$conf) {
6271         next if $opt !~ m/^unused\d+$/;
6272         my $volid = $conf->{$opt};
6273         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6274         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6275             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6276             $changes = 1;
6277             delete $conf->{$opt};
6278         }
6279
6280         $referenced->{$volid} = 1;
6281         $referencedpath->{$path} = 1 if $path;
6282     }
6283
6284     foreach my $volid (sort keys %$volid_hash) {
6285         next if $volid =~ m/vm-$vmid-state-/;
6286         next if $referenced->{$volid};
6287         my $path = $volid_hash->{$volid}->{path};
6288         next if !$path; # just to be sure
6289         next if $referencedpath->{$path};
6290         $changes = 1;
6291         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6292         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6293         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6294     }
6295
6296     return $changes;
6297 }
6298
6299 sub rescan {
6300     my ($vmid, $nolock, $dryrun) = @_;
6301
6302     my $cfg = PVE::Storage::config();
6303
6304     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6305     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6306     foreach my $stor (keys %{$cfg->{ids}}) {
6307         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6308     }
6309
6310     print "rescan volumes...\n";
6311     my $volid_hash = scan_volids($cfg, $vmid);
6312
6313     my $updatefn =  sub {
6314         my ($vmid) = @_;
6315
6316         my $conf = PVE::QemuConfig->load_config($vmid);
6317
6318         PVE::QemuConfig->check_lock($conf);
6319
6320         my $vm_volids = {};
6321         foreach my $volid (keys %$volid_hash) {
6322             my $info = $volid_hash->{$volid};
6323             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6324         }
6325
6326         my $changes = update_disksize($vmid, $conf, $vm_volids);
6327
6328         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6329     };
6330
6331     if (defined($vmid)) {
6332         if ($nolock) {
6333             &$updatefn($vmid);
6334         } else {
6335             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6336         }
6337     } else {
6338         my $vmlist = config_list();
6339         foreach my $vmid (keys %$vmlist) {
6340             if ($nolock) {
6341                 &$updatefn($vmid);
6342             } else {
6343                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6344             }
6345         }
6346     }
6347 }
6348
6349 sub restore_vma_archive {
6350     my ($archive, $vmid, $user, $opts, $comp) = @_;
6351
6352     my $readfrom = $archive;
6353
6354     my $cfg = PVE::Storage::config();
6355     my $commands = [];
6356     my $bwlimit = $opts->{bwlimit};
6357
6358     my $dbg_cmdstring = '';
6359     my $add_pipe = sub {
6360         my ($cmd) = @_;
6361         push @$commands, $cmd;
6362         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6363         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6364         $readfrom = '-';
6365     };
6366
6367     my $input = undef;
6368     if ($archive eq '-') {
6369         $input = '<&STDIN';
6370     } else {
6371         # If we use a backup from a PVE defined storage we also consider that
6372         # storage's rate limit:
6373         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6374         if (defined($volid)) {
6375             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6376             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6377             if ($readlimit) {
6378                 print STDERR "applying read rate limit: $readlimit\n";
6379                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6380                 $add_pipe->($cstream);
6381             }
6382         }
6383     }
6384
6385     if ($comp) {
6386         my $cmd;
6387         if ($comp eq 'gzip') {
6388             $cmd = ['zcat', $readfrom];
6389         } elsif ($comp eq 'lzop') {
6390             $cmd = ['lzop', '-d', '-c', $readfrom];
6391         } else {
6392             die "unknown compression method '$comp'\n";
6393         }
6394         $add_pipe->($cmd);
6395     }
6396
6397     my $tmpdir = "/var/tmp/vzdumptmp$$";
6398     rmtree $tmpdir;
6399
6400     # disable interrupts (always do cleanups)
6401     local $SIG{INT} =
6402         local $SIG{TERM} =
6403         local $SIG{QUIT} =
6404         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6405
6406     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6407     POSIX::mkfifo($mapfifo, 0600);
6408     my $fifofh;
6409
6410     my $openfifo = sub {
6411         open($fifofh, '>', $mapfifo) || die $!;
6412     };
6413
6414     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6415
6416     my $oldtimeout;
6417     my $timeout = 5;
6418
6419     my $devinfo = {};
6420
6421     my $rpcenv = PVE::RPCEnvironment::get();
6422
6423     my $conffile = PVE::QemuConfig->config_file($vmid);
6424     my $tmpfn = "$conffile.$$.tmp";
6425
6426     # Note: $oldconf is undef if VM does not exists
6427     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6428     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6429
6430     my %storage_limits;
6431
6432     my $print_devmap = sub {
6433         my $virtdev_hash = {};
6434
6435         my $cfgfn = "$tmpdir/qemu-server.conf";
6436
6437         # we can read the config - that is already extracted
6438         my $fh = IO::File->new($cfgfn, "r") ||
6439             "unable to read qemu-server.conf - $!\n";
6440
6441         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6442         if (-f $fwcfgfn) {
6443             my $pve_firewall_dir = '/etc/pve/firewall';
6444             mkdir $pve_firewall_dir; # make sure the dir exists
6445             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6446         }
6447
6448         while (defined(my $line = <$fh>)) {
6449             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6450                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6451                 die "archive does not contain data for drive '$virtdev'\n"
6452                     if !$devinfo->{$devname};
6453                 if (defined($opts->{storage})) {
6454                     $storeid = $opts->{storage} || 'local';
6455                 } elsif (!$storeid) {
6456                     $storeid = 'local';
6457                 }
6458                 $format = 'raw' if !$format;
6459                 $devinfo->{$devname}->{devname} = $devname;
6460                 $devinfo->{$devname}->{virtdev} = $virtdev;
6461                 $devinfo->{$devname}->{format} = $format;
6462                 $devinfo->{$devname}->{storeid} = $storeid;
6463
6464                 # check permission on storage
6465                 my $pool = $opts->{pool}; # todo: do we need that?
6466                 if ($user ne 'root@pam') {
6467                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6468                 }
6469
6470                 $storage_limits{$storeid} = $bwlimit;
6471
6472                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6473             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6474                 my $virtdev = $1;
6475                 my $drive = parse_drive($virtdev, $2);
6476                 if (drive_is_cloudinit($drive)) {
6477                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6478                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6479                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6480
6481                     my $d = {
6482                         format => $format,
6483                         storeid => $opts->{storage} // $storeid,
6484                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6485                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6486                         name => "vm-$vmid-cloudinit",
6487                         is_cloudinit => 1,
6488                     };
6489                     $virtdev_hash->{$virtdev} = $d;
6490                 }
6491             }
6492         }
6493
6494         foreach my $key (keys %storage_limits) {
6495             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6496             next if !$limit;
6497             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6498             $storage_limits{$key} = $limit * 1024;
6499         }
6500
6501         foreach my $devname (keys %$devinfo) {
6502             die "found no device mapping information for device '$devname'\n"
6503                 if !$devinfo->{$devname}->{virtdev};
6504         }
6505
6506         # create empty/temp config
6507         if ($oldconf) {
6508             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6509             foreach_drive($oldconf, sub {
6510                 my ($ds, $drive) = @_;
6511
6512                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6513
6514                 my $volid = $drive->{file};
6515                 return if !$volid || $volid =~ m|^/|;
6516
6517                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6518                 return if !$path || !$owner || ($owner != $vmid);
6519
6520                 # Note: only delete disk we want to restore
6521                 # other volumes will become unused
6522                 if ($virtdev_hash->{$ds}) {
6523                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6524                     if (my $err = $@) {
6525                         warn $err;
6526                     }
6527                 }
6528             });
6529
6530             # delete vmstate files, after the restore we have no snapshots anymore
6531             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6532                 my $snap = $oldconf->{snapshots}->{$snapname};
6533                 if ($snap->{vmstate}) {
6534                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6535                     if (my $err = $@) {
6536                         warn $err;
6537                     }
6538                 }
6539             }
6540         }
6541
6542         my $map = {};
6543         foreach my $virtdev (sort keys %$virtdev_hash) {
6544             my $d = $virtdev_hash->{$virtdev};
6545             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6546             my $storeid = $d->{storeid};
6547             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6548
6549             my $map_opts = '';
6550             if (my $limit = $storage_limits{$storeid}) {
6551                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6552             }
6553
6554             # test if requested format is supported
6555             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6556             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6557             $d->{format} = $defFormat if !$supported;
6558
6559             my $name;
6560             if ($d->{is_cloudinit}) {
6561                 $name = $d->{name};
6562                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6563             }
6564
6565             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6566             print STDERR "new volume ID is '$volid'\n";
6567             $d->{volid} = $volid;
6568
6569             PVE::Storage::activate_volumes($cfg, [$volid]);
6570
6571             my $write_zeros = 1;
6572             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6573                 $write_zeros = 0;
6574             }
6575
6576             if (!$d->{is_cloudinit}) {
6577                 my $path = PVE::Storage::path($cfg, $volid);
6578
6579                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6580
6581                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6582             }
6583             $map->{$virtdev} = $volid;
6584         }
6585
6586         $fh->seek(0, 0) || die "seek failed - $!\n";
6587
6588         my $outfd = new IO::File ($tmpfn, "w") ||
6589             die "unable to write config for VM $vmid\n";
6590
6591         my $cookie = { netcount => 0 };
6592         while (defined(my $line = <$fh>)) {
6593             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6594         }
6595
6596         $fh->close();
6597         $outfd->close();
6598     };
6599
6600     eval {
6601         # enable interrupts
6602         local $SIG{INT} =
6603             local $SIG{TERM} =
6604             local $SIG{QUIT} =
6605             local $SIG{HUP} =
6606             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6607         local $SIG{ALRM} = sub { die "got timeout\n"; };
6608
6609         $oldtimeout = alarm($timeout);
6610
6611         my $parser = sub {
6612             my $line = shift;
6613
6614             print "$line\n";
6615
6616             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6617                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6618                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6619             } elsif ($line =~ m/^CTIME: /) {
6620                 # we correctly received the vma config, so we can disable
6621                 # the timeout now for disk allocation (set to 10 minutes, so
6622                 # that we always timeout if something goes wrong)
6623                 alarm(600);
6624                 &$print_devmap();
6625                 print $fifofh "done\n";
6626                 my $tmp = $oldtimeout || 0;
6627                 $oldtimeout = undef;
6628                 alarm($tmp);
6629                 close($fifofh);
6630             }
6631         };
6632
6633         print "restore vma archive: $dbg_cmdstring\n";
6634         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6635     };
6636     my $err = $@;
6637
6638     alarm($oldtimeout) if $oldtimeout;
6639
6640     my $vollist = [];
6641     foreach my $devname (keys %$devinfo) {
6642         my $volid = $devinfo->{$devname}->{volid};
6643         push @$vollist, $volid if $volid;
6644     }
6645
6646     PVE::Storage::deactivate_volumes($cfg, $vollist);
6647
6648     unlink $mapfifo;
6649
6650     if ($err) {
6651         rmtree $tmpdir;
6652         unlink $tmpfn;
6653
6654         foreach my $devname (keys %$devinfo) {
6655             my $volid = $devinfo->{$devname}->{volid};
6656             next if !$volid;
6657             eval {
6658                 if ($volid =~ m|^/|) {
6659                     unlink $volid || die 'unlink failed\n';
6660                 } else {
6661                     PVE::Storage::vdisk_free($cfg, $volid);
6662                 }
6663                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6664             };
6665             print STDERR "unable to cleanup '$volid' - $@" if $@;
6666         }
6667         die $err;
6668     }
6669
6670     rmtree $tmpdir;
6671
6672     rename($tmpfn, $conffile) ||
6673         die "unable to commit configuration file '$conffile'\n";
6674
6675     PVE::Cluster::cfs_update(); # make sure we read new file
6676
6677     eval { rescan($vmid, 1); };
6678     warn $@ if $@;
6679 }
6680
6681 sub restore_tar_archive {
6682     my ($archive, $vmid, $user, $opts) = @_;
6683
6684     if ($archive ne '-') {
6685         my $firstfile = tar_archive_read_firstfile($archive);
6686         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6687             if $firstfile ne 'qemu-server.conf';
6688     }
6689
6690     my $storecfg = PVE::Storage::config();
6691
6692     # destroy existing data - keep empty config
6693     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6694     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6695
6696     my $tocmd = "/usr/lib/qemu-server/qmextract";
6697
6698     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6699     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6700     $tocmd .= ' --prealloc' if $opts->{prealloc};
6701     $tocmd .= ' --info' if $opts->{info};
6702
6703     # tar option "xf" does not autodetect compression when read from STDIN,
6704     # so we pipe to zcat
6705     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6706         PVE::Tools::shellquote("--to-command=$tocmd");
6707
6708     my $tmpdir = "/var/tmp/vzdumptmp$$";
6709     mkpath $tmpdir;
6710
6711     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6712     local $ENV{VZDUMP_VMID} = $vmid;
6713     local $ENV{VZDUMP_USER} = $user;
6714
6715     my $conffile = PVE::QemuConfig->config_file($vmid);
6716     my $tmpfn = "$conffile.$$.tmp";
6717
6718     # disable interrupts (always do cleanups)
6719     local $SIG{INT} =
6720         local $SIG{TERM} =
6721         local $SIG{QUIT} =
6722         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6723
6724     eval {
6725         # enable interrupts
6726         local $SIG{INT} =
6727             local $SIG{TERM} =
6728             local $SIG{QUIT} =
6729             local $SIG{HUP} =
6730             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6731
6732         if ($archive eq '-') {
6733             print "extracting archive from STDIN\n";
6734             run_command($cmd, input => "<&STDIN");
6735         } else {
6736             print "extracting archive '$archive'\n";
6737             run_command($cmd);
6738         }
6739
6740         return if $opts->{info};
6741
6742         # read new mapping
6743         my $map = {};
6744         my $statfile = "$tmpdir/qmrestore.stat";
6745         if (my $fd = IO::File->new($statfile, "r")) {
6746             while (defined (my $line = <$fd>)) {
6747                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6748                     $map->{$1} = $2 if $1;
6749                 } else {
6750                     print STDERR "unable to parse line in statfile - $line\n";
6751                 }
6752             }
6753             $fd->close();
6754         }
6755
6756         my $confsrc = "$tmpdir/qemu-server.conf";
6757
6758         my $srcfd = new IO::File($confsrc, "r") ||
6759             die "unable to open file '$confsrc'\n";
6760
6761         my $outfd = new IO::File ($tmpfn, "w") ||
6762             die "unable to write config for VM $vmid\n";
6763
6764         my $cookie = { netcount => 0 };
6765         while (defined (my $line = <$srcfd>)) {
6766             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6767         }
6768
6769         $srcfd->close();
6770         $outfd->close();
6771     };
6772     my $err = $@;
6773
6774     if ($err) {
6775
6776         unlink $tmpfn;
6777
6778         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6779
6780         die $err;
6781     }
6782
6783     rmtree $tmpdir;
6784
6785     rename $tmpfn, $conffile ||
6786         die "unable to commit configuration file '$conffile'\n";
6787
6788     PVE::Cluster::cfs_update(); # make sure we read new file
6789
6790     eval { rescan($vmid, 1); };
6791     warn $@ if $@;
6792 };
6793
6794 sub foreach_storage_used_by_vm {
6795     my ($conf, $func) = @_;
6796
6797     my $sidhash = {};
6798
6799     foreach_drive($conf, sub {
6800         my ($ds, $drive) = @_;
6801         return if drive_is_cdrom($drive);
6802
6803         my $volid = $drive->{file};
6804
6805         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6806         $sidhash->{$sid} = $sid if $sid;
6807     });
6808
6809     foreach my $sid (sort keys %$sidhash) {
6810         &$func($sid);
6811     }
6812 }
6813
6814 sub do_snapshots_with_qemu {
6815     my ($storecfg, $volid) = @_;
6816
6817     my $storage_name = PVE::Storage::parse_volume_id($volid);
6818     my $scfg = $storecfg->{ids}->{$storage_name};
6819
6820     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6821         return 1;
6822     }
6823
6824     if ($volid =~ m/\.(qcow2|qed)$/){
6825         return 1;
6826     }
6827
6828     return undef;
6829 }
6830
6831 sub qga_check_running {
6832     my ($vmid, $nowarn) = @_;
6833
6834     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6835     if ($@) {
6836         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6837         return 0;
6838     }
6839     return 1;
6840 }
6841
6842 sub template_create {
6843     my ($vmid, $conf, $disk) = @_;
6844
6845     my $storecfg = PVE::Storage::config();
6846
6847     foreach_drive($conf, sub {
6848         my ($ds, $drive) = @_;
6849
6850         return if drive_is_cdrom($drive);
6851         return if $disk && $ds ne $disk;
6852
6853         my $volid = $drive->{file};
6854         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6855
6856         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6857         $drive->{file} = $voliddst;
6858         $conf->{$ds} = print_drive($vmid, $drive);
6859         PVE::QemuConfig->write_config($vmid, $conf);
6860     });
6861 }
6862
6863 sub convert_iscsi_path {
6864     my ($path) = @_;
6865
6866     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6867         my $portal = $1;
6868         my $target = $2;
6869         my $lun = $3;
6870
6871         my $initiator_name = get_initiator_name();
6872
6873         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6874                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6875     }
6876
6877     die "cannot convert iscsi path '$path', unkown format\n";
6878 }
6879
6880 sub qemu_img_convert {
6881     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6882
6883     my $storecfg = PVE::Storage::config();
6884     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6885     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6886
6887     if ($src_storeid && $dst_storeid) {
6888
6889         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6890
6891         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6892         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6893
6894         my $src_format = qemu_img_format($src_scfg, $src_volname);
6895         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6896
6897         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6898         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6899
6900         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6901         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6902
6903         my $cmd = [];
6904         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6905         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6906         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6907         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6908
6909         if ($src_is_iscsi) {
6910             push @$cmd, '--image-opts';
6911             $src_path = convert_iscsi_path($src_path);
6912         } else {
6913             push @$cmd, '-f', $src_format;
6914         }
6915
6916         if ($dst_is_iscsi) {
6917             push @$cmd, '--target-image-opts';
6918             $dst_path = convert_iscsi_path($dst_path);
6919         } else {
6920             push @$cmd, '-O', $dst_format;
6921         }
6922
6923         push @$cmd, $src_path;
6924
6925         if (!$dst_is_iscsi && $is_zero_initialized) {
6926             push @$cmd, "zeroinit:$dst_path";
6927         } else {
6928             push @$cmd, $dst_path;
6929         }
6930
6931         my $parser = sub {
6932             my $line = shift;
6933             if($line =~ m/\((\S+)\/100\%\)/){
6934                 my $percent = $1;
6935                 my $transferred = int($size * $percent / 100);
6936                 my $remaining = $size - $transferred;
6937
6938                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6939             }
6940
6941         };
6942
6943         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6944         my $err = $@;
6945         die "copy failed: $err" if $err;
6946     }
6947 }
6948
6949 sub qemu_img_format {
6950     my ($scfg, $volname) = @_;
6951
6952     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6953         return $1;
6954     } else {
6955         return "raw";
6956     }
6957 }
6958
6959 sub qemu_drive_mirror {
6960     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6961
6962     $jobs = {} if !$jobs;
6963
6964     my $qemu_target;
6965     my $format;
6966     $jobs->{"drive-$drive"} = {};
6967
6968     if ($dst_volid =~ /^nbd:/) {
6969         $qemu_target = $dst_volid;
6970         $format = "nbd";
6971     } else {
6972         my $storecfg = PVE::Storage::config();
6973         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6974
6975         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6976
6977         $format = qemu_img_format($dst_scfg, $dst_volname);
6978
6979         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6980
6981         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6982     }
6983
6984     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6985     $opts->{format} = $format if $format;
6986
6987     if (defined($bwlimit)) {
6988         $opts->{speed} = $bwlimit * 1024;
6989         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6990     } else {
6991         print "drive mirror is starting for drive-$drive\n";
6992     }
6993
6994     # if a job already runs for this device we get an error, catch it for cleanup
6995     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6996     if (my $err = $@) {
6997         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6998         warn "$@\n" if $@;
6999         die "mirroring error: $err\n";
7000     }
7001
7002     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7003 }
7004
7005 sub qemu_drive_mirror_monitor {
7006     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7007
7008     eval {
7009         my $err_complete = 0;
7010
7011         while (1) {
7012             die "storage migration timed out\n" if $err_complete > 300;
7013
7014             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7015
7016             my $running_mirror_jobs = {};
7017             foreach my $stat (@$stats) {
7018                 next if $stat->{type} ne 'mirror';
7019                 $running_mirror_jobs->{$stat->{device}} = $stat;
7020             }
7021
7022             my $readycounter = 0;
7023
7024             foreach my $job (keys %$jobs) {
7025
7026                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7027                     print "$job : finished\n";
7028                     delete $jobs->{$job};
7029                     next;
7030                 }
7031
7032                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7033
7034                 my $busy = $running_mirror_jobs->{$job}->{busy};
7035                 my $ready = $running_mirror_jobs->{$job}->{ready};
7036                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7037                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7038                     my $remaining = $total - $transferred;
7039                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7040
7041                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7042                 }
7043
7044                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7045             }
7046
7047             last if scalar(keys %$jobs) == 0;
7048
7049             if ($readycounter == scalar(keys %$jobs)) {
7050                 print "all mirroring jobs are ready \n";
7051                 last if $skipcomplete; #do the complete later
7052
7053                 if ($vmiddst && $vmiddst != $vmid) {
7054                     my $agent_running = $qga && qga_check_running($vmid);
7055                     if ($agent_running) {
7056                         print "freeze filesystem\n";
7057                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7058                     } else {
7059                         print "suspend vm\n";
7060                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7061                     }
7062
7063                     # if we clone a disk for a new target vm, we don't switch the disk
7064                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7065
7066                     if ($agent_running) {
7067                         print "unfreeze filesystem\n";
7068                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7069                     } else {
7070                         print "resume vm\n";
7071                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7072                     }
7073
7074                     last;
7075                 } else {
7076
7077                     foreach my $job (keys %$jobs) {
7078                         # try to switch the disk if source and destination are on the same guest
7079                         print "$job: Completing block job...\n";
7080
7081                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7082                         if ($@ =~ m/cannot be completed/) {
7083                             print "$job: Block job cannot be completed, try again.\n";
7084                             $err_complete++;
7085                         }else {
7086                             print "$job: Completed successfully.\n";
7087                             $jobs->{$job}->{complete} = 1;
7088                         }
7089                     }
7090                 }
7091             }
7092             sleep 1;
7093         }
7094     };
7095     my $err = $@;
7096
7097     if ($err) {
7098         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7099         die "mirroring error: $err";
7100     }
7101
7102 }
7103
7104 sub qemu_blockjobs_cancel {
7105     my ($vmid, $jobs) = @_;
7106
7107     foreach my $job (keys %$jobs) {
7108         print "$job: Cancelling block job\n";
7109         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7110         $jobs->{$job}->{cancel} = 1;
7111     }
7112
7113     while (1) {
7114         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7115
7116         my $running_jobs = {};
7117         foreach my $stat (@$stats) {
7118             $running_jobs->{$stat->{device}} = $stat;
7119         }
7120
7121         foreach my $job (keys %$jobs) {
7122
7123             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7124                 print "$job: Done.\n";
7125                 delete $jobs->{$job};
7126             }
7127         }
7128
7129         last if scalar(keys %$jobs) == 0;
7130
7131         sleep 1;
7132     }
7133 }
7134
7135 sub clone_disk {
7136     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7137         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7138
7139     my $newvolid;
7140
7141     if (!$full) {
7142         print "create linked clone of drive $drivename ($drive->{file})\n";
7143         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7144         push @$newvollist, $newvolid;
7145     } else {
7146
7147         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7148         $storeid = $storage if $storage;
7149
7150         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7151         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7152
7153         print "create full clone of drive $drivename ($drive->{file})\n";
7154         my $name = undef;
7155         if (drive_is_cloudinit($drive)) {
7156             $name = "vm-$newvmid-cloudinit";
7157             $snapname = undef;
7158             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7159             if ($dst_format ne 'raw') {
7160                 $name .= ".$dst_format";
7161             }
7162         }
7163         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7164         push @$newvollist, $newvolid;
7165
7166         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7167
7168         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7169         if (!$running || $snapname) {
7170             # TODO: handle bwlimits
7171             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7172         } else {
7173
7174             my $kvmver = get_running_qemu_version ($vmid);
7175             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7176                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7177                     if $drive->{iothread};
7178             }
7179
7180             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7181         }
7182     }
7183
7184     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7185
7186     my $disk = $drive;
7187     $disk->{format} = undef;
7188     $disk->{file} = $newvolid;
7189     $disk->{size} = $size;
7190
7191     return $disk;
7192 }
7193
7194 # this only works if VM is running
7195 sub get_current_qemu_machine {
7196     my ($vmid) = @_;
7197
7198     my $cmd = { execute => 'query-machines', arguments => {} };
7199     my $res = vm_qmp_command($vmid, $cmd);
7200
7201     my ($current, $default);
7202     foreach my $e (@$res) {
7203         $default = $e->{name} if $e->{'is-default'};
7204         $current = $e->{name} if $e->{'is-current'};
7205     }
7206
7207     # fallback to the default machine if current is not supported by qemu
7208     return $current || $default || 'pc';
7209 }
7210
7211 sub get_running_qemu_version {
7212     my ($vmid) = @_;
7213     my $cmd = { execute => 'query-version', arguments => {} };
7214     my $res = vm_qmp_command($vmid, $cmd);
7215     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7216 }
7217
7218 sub qemu_machine_feature_enabled {
7219     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7220
7221     my $current_major;
7222     my $current_minor;
7223
7224     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7225
7226         $current_major = $3;
7227         $current_minor = $4;
7228
7229     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7230
7231         $current_major = $1;
7232         $current_minor = $2;
7233     }
7234
7235     return 1 if $current_major > $version_major ||
7236                 ($current_major == $version_major &&
7237                  $current_minor >= $version_minor);
7238 }
7239
7240 sub qemu_machine_pxe {
7241     my ($vmid, $conf) = @_;
7242
7243     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7244
7245     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7246         $machine .= '.pxe';
7247     }
7248
7249     return $machine;
7250 }
7251
7252 sub qemu_use_old_bios_files {
7253     my ($machine_type) = @_;
7254
7255     return if !$machine_type;
7256
7257     my $use_old_bios_files = undef;
7258
7259     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7260         $machine_type = $1;
7261         $use_old_bios_files = 1;
7262     } else {
7263         my $kvmver = kvm_user_version();
7264         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7265         # load new efi bios files on migration. So this hack is required to allow
7266         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7267         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7268         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7269     }
7270
7271     return ($use_old_bios_files, $machine_type);
7272 }
7273
7274 sub create_efidisk($$$$$) {
7275     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7276
7277     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7278     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7279
7280     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7281     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7282     PVE::Storage::activate_volumes($storecfg, [$volid]);
7283
7284     my $path = PVE::Storage::path($storecfg, $volid);
7285     eval {
7286         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7287     };
7288     die "Copying EFI vars image failed: $@" if $@;
7289
7290     return ($volid, $vars_size);
7291 }
7292
7293 sub vm_iothreads_list {
7294     my ($vmid) = @_;
7295
7296     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7297
7298     my $iothreads = {};
7299     foreach my $iothread (@$res) {
7300         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7301     }
7302
7303     return $iothreads;
7304 }
7305
7306 sub scsihw_infos {
7307     my ($conf, $drive) = @_;
7308
7309     my $maxdev = 0;
7310
7311     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7312         $maxdev = 7;
7313     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7314         $maxdev = 1;
7315     } else {
7316         $maxdev = 256;
7317     }
7318
7319     my $controller = int($drive->{index} / $maxdev);
7320     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7321
7322     return ($maxdev, $controller, $controller_prefix);
7323 }
7324
7325 sub add_hyperv_enlightenments {
7326     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7327
7328     return if $winversion < 6;
7329     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7330
7331     if ($gpu_passthrough || defined($hv_vendor_id)) {
7332         $hv_vendor_id //= 'proxmox';
7333         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7334     }
7335
7336     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7337         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7338         push @$cpuFlags , 'hv_vapic';
7339         push @$cpuFlags , 'hv_time';
7340     } else {
7341         push @$cpuFlags , 'hv_spinlocks=0xffff';
7342     }
7343
7344     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7345         push @$cpuFlags , 'hv_reset';
7346         push @$cpuFlags , 'hv_vpindex';
7347         push @$cpuFlags , 'hv_runtime';
7348     }
7349
7350     if ($winversion >= 7) {
7351         push @$cpuFlags , 'hv_relaxed';
7352
7353         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7354             push @$cpuFlags , 'hv_synic';
7355             push @$cpuFlags , 'hv_stimer';
7356         }
7357
7358         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7359             push @$cpuFlags , 'hv_ipi';
7360         }
7361     }
7362 }
7363
7364 sub windows_version {
7365     my ($ostype) = @_;
7366
7367     return 0 if !$ostype;
7368
7369     my $winversion = 0;
7370
7371     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7372         $winversion = 5;
7373     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7374         $winversion = 6;
7375     } elsif ($ostype =~ m/^win(\d+)$/) {
7376         $winversion = $1;
7377     }
7378
7379     return $winversion;
7380 }
7381
7382 sub resolve_dst_disk_format {
7383         my ($storecfg, $storeid, $src_volname, $format) = @_;
7384         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7385
7386         if (!$format) {
7387             # if no target format is specified, use the source disk format as hint
7388             if ($src_volname) {
7389                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7390                 $format = qemu_img_format($scfg, $src_volname);
7391             } else {
7392                 return $defFormat;
7393             }
7394         }
7395
7396         # test if requested format is supported - else use default
7397         my $supported = grep { $_ eq $format } @$validFormats;
7398         $format = $defFormat if !$supported;
7399         return $format;
7400 }
7401
7402 sub resolve_first_disk {
7403     my $conf = shift;
7404     my @disks = PVE::QemuServer::valid_drive_names();
7405     my $firstdisk;
7406     foreach my $ds (reverse @disks) {
7407         next if !$conf->{$ds};
7408         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7409         next if PVE::QemuServer::drive_is_cdrom($disk);
7410         $firstdisk = $ds;
7411     }
7412     return $firstdisk;
7413 }
7414
7415 sub generate_uuid {
7416     my ($uuid, $uuid_str);
7417     UUID::generate($uuid);
7418     UUID::unparse($uuid, $uuid_str);
7419     return $uuid_str;
7420 }
7421
7422 sub generate_smbios1_uuid {
7423     return "uuid=".generate_uuid();
7424 }
7425
7426 sub nbd_stop {
7427     my ($vmid) = @_;
7428
7429     vm_mon_cmd($vmid, 'nbd-server-stop');
7430 }
7431
7432 sub create_reboot_request {
7433     my ($vmid) = @_;
7434     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7435         or die "failed to create reboot trigger file: $!\n";
7436     close($fh);
7437 }
7438
7439 sub clear_reboot_request {
7440     my ($vmid) = @_;
7441     my $path = "/run/qemu-server/$vmid.reboot";
7442     my $res = 0;
7443
7444     $res = unlink($path);
7445     die "could not remove reboot request for $vmid: $!"
7446         if !$res && $! != POSIX::ENOENT;
7447
7448     return $res;
7449 }
7450
7451 # bash completion helper
7452
7453 sub complete_backup_archives {
7454     my ($cmdname, $pname, $cvalue) = @_;
7455
7456     my $cfg = PVE::Storage::config();
7457
7458     my $storeid;
7459
7460     if ($cvalue =~ m/^([^:]+):/) {
7461         $storeid = $1;
7462     }
7463
7464     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7465
7466     my $res = [];
7467     foreach my $id (keys %$data) {
7468         foreach my $item (@{$data->{$id}}) {
7469             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7470             push @$res, $item->{volid} if defined($item->{volid});
7471         }
7472     }
7473
7474     return $res;
7475 }
7476
7477 my $complete_vmid_full = sub {
7478     my ($running) = @_;
7479
7480     my $idlist = vmstatus();
7481
7482     my $res = [];
7483
7484     foreach my $id (keys %$idlist) {
7485         my $d = $idlist->{$id};
7486         if (defined($running)) {
7487             next if $d->{template};
7488             next if $running && $d->{status} ne 'running';
7489             next if !$running && $d->{status} eq 'running';
7490         }
7491         push @$res, $id;
7492
7493     }
7494     return $res;
7495 };
7496
7497 sub complete_vmid {
7498     return &$complete_vmid_full();
7499 }
7500
7501 sub complete_vmid_stopped {
7502     return &$complete_vmid_full(0);
7503 }
7504
7505 sub complete_vmid_running {
7506     return &$complete_vmid_full(1);
7507 }
7508
7509 sub complete_storage {
7510
7511     my $cfg = PVE::Storage::config();
7512     my $ids = $cfg->{ids};
7513
7514     my $res = [];
7515     foreach my $sid (keys %$ids) {
7516         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7517         next if !$ids->{$sid}->{content}->{images};
7518         push @$res, $sid;
7519     }
7520
7521     return $res;
7522 }
7523
7524 1;