PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne typesafe_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  55 my $OVMF = {
  56     x86_64 => [
  57         "$EDK2_FW_BASE/OVMF_CODE.fd",
  58         "$EDK2_FW_BASE/OVMF_VARS.fd"
  59     ],
  60     aarch64 => [
  61         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  62         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  63     ],
  64 };
  65
  66 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  87         description => "Specifies the Qemu machine type.",
  88         type => 'string',
  89         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  90         maxLength => 40,
  91         optional => 1,
  92 });
  93
  94 #no warnings 'redefine';
  95
  96 sub cgroups_write {
  97    my ($controller, $vmid, $option, $value) = @_;
  98
  99    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 100    PVE::ProcFSTools::write_proc_entry($path, $value);
 101
 102 }
 103
 104 my $nodename_cache;
 105 sub nodename {
 106     $nodename_cache //= PVE::INotify::nodename();
 107     return $nodename_cache;
 108 }
 109
 110 my $watchdog_fmt = {
 111     model => {
 112         default_key => 1,
 113         type => 'string',
 114         enum => [qw(i6300esb ib700)],
 115         description => "Watchdog type to emulate.",
 116         default => 'i6300esb',
 117         optional => 1,
 118     },
 119     action => {
 120         type => 'string',
 121         enum => [qw(reset shutdown poweroff pause debug none)],
 122         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 123         optional => 1,
 124     },
 125 };
 126 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 127
 128 my $agent_fmt = {
 129     enabled => {
 130         description => "Enable/disable Qemu GuestAgent.",
 131         type => 'boolean',
 132         default => 0,
 133         default_key => 1,
 134     },
 135     fstrim_cloned_disks => {
 136         description => "Run fstrim after cloning/moving a disk.",
 137         type => 'boolean',
 138         optional => 1,
 139         default => 0
 140     },
 141     type => {
 142         description => "Select the agent type",
 143         type => 'string',
 144         default => 'virtio',
 145         optional => 1,
 146         enum => [qw(virtio isa)],
 147     },
 148 };
 149
 150 my $vga_fmt = {
 151     type => {
 152         description => "Select the VGA type.",
 153         type => 'string',
 154         default => 'std',
 155         optional => 1,
 156         default_key => 1,
 157         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 158     },
 159     memory => {
 160         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 161         type => 'integer',
 162         optional => 1,
 163         minimum => 4,
 164         maximum => 512,
 165     },
 166 };
 167
 168 my $ivshmem_fmt = {
 169     size => {
 170         type => 'integer',
 171         minimum => 1,
 172         description => "The size of the file in MB.",
 173     },
 174     name => {
 175         type => 'string',
 176         pattern => '[a-zA-Z0-9\-]+',
 177         optional => 1,
 178         format_description => 'string',
 179         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 180     },
 181 };
 182
 183 my $audio_fmt = {
 184     device => {
 185         type => 'string',
 186         enum => [qw(ich9-intel-hda intel-hda AC97)],
 187         description =>  "Configure an audio device."
 188     },
 189     driver =>  {
 190         type => 'string',
 191         enum => ['spice'],
 192         default => 'spice',
 193         optional => 1,
 194         description => "Driver backend for the audio device."
 195     },
 196 };
 197
 198 my $spice_enhancements_fmt = {
 199     foldersharing => {
 200         type => 'boolean',
 201         optional => 1,
 202         default => '0',
 203         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 204     },
 205     videostreaming =>  {
 206         type => 'string',
 207         enum => ['off', 'all', 'filter'],
 208         default => 'off',
 209         optional => 1,
 210         description => "Enable video streaming. Uses compression for detected video streams."
 211     },
 212 };
 213
 214 my $rng_fmt = {
 215     source => {
 216         type => 'string',
 217         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 218         default_key => 1,
 219         description => "The file on the host to gather entropy from. In most"
 220                      . " cases /dev/urandom should be preferred over /dev/random"
 221                      . " to avoid entropy-starvation issues on the host. Using"
 222                      . " urandom does *not* decrease security in any meaningful"
 223                      . " way, as it's still seeded from real entropy, and the"
 224                      . " bytes provided will most likely be mixed with real"
 225                      . " entropy on the guest as well. /dev/hwrng can be used"
 226                      . " to pass through a hardware RNG from the host.",
 227     },
 228     max_bytes => {
 229         type => 'integer',
 230         description => "Maximum bytes of entropy injected into the guest every"
 231                      . " 'period' milliseconds. Prefer a lower value when using"
 232                      . " /dev/random as source. Use 0 to disable limiting"
 233                      . " (potentially dangerous!).",
 234         optional => 1,
 235
 236         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 237         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 238         # of overwhelming the host, provided we're reading from /dev/urandom
 239         default => 1024,
 240     },
 241     period => {
 242         type => 'integer',
 243         description => "Every 'period' milliseconds the entropy-injection quota"
 244                      . " is reset, allowing the guest to retrieve another"
 245                      . " 'max_bytes' of entropy.",
 246         optional => 1,
 247         default => 1000,
 248     },
 249 };
 250
 251 my $confdesc = {
 252     onboot => {
 253         optional => 1,
 254         type => 'boolean',
 255         description => "Specifies whether a VM will be started during system bootup.",
 256         default => 0,
 257     },
 258     autostart => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Automatic restart after crash (currently ignored).",
 262         default => 0,
 263     },
 264     hotplug => {
 265         optional => 1,
 266         type => 'string', format => 'pve-hotplug-features',
 267         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 268         default => 'network,disk,usb',
 269     },
 270     reboot => {
 271         optional => 1,
 272         type => 'boolean',
 273         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 274         default => 1,
 275     },
 276     lock => {
 277         optional => 1,
 278         type => 'string',
 279         description => "Lock/unlock the VM.",
 280         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 281     },
 282     cpulimit => {
 283         optional => 1,
 284         type => 'number',
 285         description => "Limit of CPU usage.",
 286         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 287         minimum => 0,
 288         maximum => 128,
 289         default => 0,
 290     },
 291     cpuunits => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "CPU weight for a VM.",
 295         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 296         minimum => 2,
 297         maximum => 262144,
 298         default => 1024,
 299     },
 300     memory => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 304         minimum => 16,
 305         default => 512,
 306     },
 307     balloon => {
 308         optional => 1,
 309         type => 'integer',
 310         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 311         minimum => 0,
 312     },
 313     shares => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 317         minimum => 0,
 318         maximum => 50000,
 319         default => 1000,
 320     },
 321     keyboard => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 325                        "It should not be necessary to set it.",
 326         enum => PVE::Tools::kvmkeymaplist(),
 327         default => undef,
 328     },
 329     name => {
 330         optional => 1,
 331         type => 'string', format => 'dns-name',
 332         description => "Set a name for the VM. Only used on the configuration web interface.",
 333     },
 334     scsihw => {
 335         optional => 1,
 336         type => 'string',
 337         description => "SCSI controller model",
 338         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 339         default => 'lsi',
 340     },
 341     description => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 345     },
 346     ostype => {
 347         optional => 1,
 348         type => 'string',
 349         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 350         description => "Specify guest operating system.",
 351         verbose_description => <<EODESC,
 352 Specify guest operating system. This is used to enable special
 353 optimization/features for specific operating systems:
 354
 355 [horizontal]
 356 other;; unspecified OS
 357 wxp;; Microsoft Windows XP
 358 w2k;; Microsoft Windows 2000
 359 w2k3;; Microsoft Windows 2003
 360 w2k8;; Microsoft Windows 2008
 361 wvista;; Microsoft Windows Vista
 362 win7;; Microsoft Windows 7
 363 win8;; Microsoft Windows 8/2012/2012r2
 364 win10;; Microsoft Windows 10/2016
 365 l24;; Linux 2.4 Kernel
 366 l26;; Linux 2.6 - 5.X Kernel
 367 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 368 EODESC
 369     },
 370     boot => {
 371         optional => 1,
 372         type => 'string',
 373         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 374         pattern => '[acdn]{1,4}',
 375         default => 'cdn',
 376     },
 377     bootdisk => {
 378         optional => 1,
 379         type => 'string', format => 'pve-qm-bootdisk',
 380         description => "Enable booting from specified disk.",
 381         pattern => '(ide|sata|scsi|virtio)\d+',
 382     },
 383     smp => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPUs. Please use option -sockets instead.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     sockets => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of CPU sockets.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     cores => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "The number of cores per socket.",
 401         minimum => 1,
 402         default => 1,
 403     },
 404     numa => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Enable/disable NUMA.",
 408         default => 0,
 409     },
 410     hugepages => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Enable/disable hugepages memory.",
 414         enum => [qw(any 2 1024)],
 415     },
 416     vcpus => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "Number of hotplugged vcpus.",
 420         minimum => 1,
 421         default => 0,
 422     },
 423     acpi => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable ACPI.",
 427         default => 1,
 428     },
 429     agent => {
 430         optional => 1,
 431         description => "Enable/disable Qemu GuestAgent and its properties.",
 432         type => 'string',
 433         format => $agent_fmt,
 434     },
 435     kvm => {
 436         optional => 1,
 437         type => 'boolean',
 438         description => "Enable/disable KVM hardware virtualization.",
 439         default => 1,
 440     },
 441     tdf => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable time drift fix.",
 445         default => 0,
 446     },
 447     localtime => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 451     },
 452     freeze => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 456     },
 457     vga => {
 458         optional => 1,
 459         type => 'string', format => $vga_fmt,
 460         description => "Configure the VGA hardware.",
 461         verbose_description => "Configure the VGA Hardware. If you want to use ".
 462             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 463             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 464             "is 'std' for all OS types besides some Windows versions (XP and " .
 465             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 466             "display server. For win* OS you can select how many independent " .
 467             "displays you want, Linux guests can add displays them self.\n".
 468             "You can also run without any graphic card, using a serial device as terminal.",
 469     },
 470     watchdog => {
 471         optional => 1,
 472         type => 'string', format => 'pve-qm-watchdog',
 473         description => "Create a virtual hardware watchdog device.",
 474         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 475             " (by a guest action), the watchdog must be periodically polled " .
 476             "by an agent inside the guest or else the watchdog will reset " .
 477             "the guest (or execute the respective action specified)",
 478     },
 479     startdate => {
 480         optional => 1,
 481         type => 'string',
 482         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 483         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 484         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 485         default => 'now',
 486     },
 487     startup =>  get_standard_option('pve-startup-order'),
 488     template => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable Template.",
 492         default => 0,
 493     },
 494     args => {
 495         optional => 1,
 496         type => 'string',
 497         description => "Arbitrary arguments passed to kvm.",
 498         verbose_description => <<EODESCR,
 499 Arbitrary arguments passed to kvm, for example:
 500
 501 args: -no-reboot -no-hpet
 502
 503 NOTE: this option is for experts only.
 504 EODESCR
 505     },
 506     tablet => {
 507         optional => 1,
 508         type => 'boolean',
 509         default => 1,
 510         description => "Enable/disable the USB tablet device.",
 511         verbose_description => "Enable/disable the USB tablet device. This device is " .
 512             "usually needed to allow absolute mouse positioning with VNC. " .
 513             "Else the mouse runs out of sync with normal VNC clients. " .
 514             "If you're running lots of console-only guests on one host, " .
 515             "you may consider disabling this to save some context switches. " .
 516             "This is turned off by default if you use spice (-vga=qxl).",
 517     },
 518     migrate_speed => {
 519         optional => 1,
 520         type => 'integer',
 521         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 522         minimum => 0,
 523         default => 0,
 524     },
 525     migrate_downtime => {
 526         optional => 1,
 527         type => 'number',
 528         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 529         minimum => 0,
 530         default => 0.1,
 531     },
 532     cdrom => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-ide',
 535         typetext => '<volume>',
 536         description => "This is an alias for option -ide2",
 537     },
 538     cpu => {
 539         optional => 1,
 540         description => "Emulated CPU type.",
 541         type => 'string',
 542         format => $PVE::QemuServer::CPUConfig::cpu_fmt,
 543     },
 544     parent => get_standard_option('pve-snapshot-name', {
 545         optional => 1,
 546         description => "Parent snapshot name. This is used internally, and should not be modified.",
 547     }),
 548     snaptime => {
 549         optional => 1,
 550         description => "Timestamp for snapshots.",
 551         type => 'integer',
 552         minimum => 0,
 553     },
 554     vmstate => {
 555         optional => 1,
 556         type => 'string', format => 'pve-volume-id',
 557         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 558     },
 559     vmstatestorage => get_standard_option('pve-storage-id', {
 560         description => "Default storage for VM state volumes/files.",
 561         optional => 1,
 562     }),
 563     runningmachine => get_standard_option('pve-qemu-machine', {
 564         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 565     }),
 566     machine => get_standard_option('pve-qemu-machine'),
 567     arch => {
 568         description => "Virtual processor architecture. Defaults to the host.",
 569         optional => 1,
 570         type => 'string',
 571         enum => [qw(x86_64 aarch64)],
 572     },
 573     smbios1 => {
 574         description => "Specify SMBIOS type 1 fields.",
 575         type => 'string', format => 'pve-qm-smbios1',
 576         maxLength => 512,
 577         optional => 1,
 578     },
 579     protection => {
 580         optional => 1,
 581         type => 'boolean',
 582         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 583         default => 0,
 584     },
 585     bios => {
 586         optional => 1,
 587         type => 'string',
 588         enum => [ qw(seabios ovmf) ],
 589         description => "Select BIOS implementation.",
 590         default => 'seabios',
 591     },
 592     vmgenid => {
 593         type => 'string',
 594         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 595         format_description => 'UUID',
 596         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 597         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 598             " 128-bit integer value identifier to the guest OS. This allows to".
 599             " notify the guest operating system when the virtual machine is".
 600             " executed with a different configuration (e.g. snapshot execution".
 601             " or creation from a template). The guest operating system notices".
 602             " the change, and is then able to react as appropriate by marking".
 603             " its copies of distributed databases as dirty, re-initializing its".
 604             " random number generator, etc.\n".
 605             "Note that auto-creation only works when done throug API/CLI create".
 606             " or update methods, but not when manually editing the config file.",
 607         default => "1 (autogenerated)",
 608         optional => 1,
 609     },
 610     hookscript => {
 611         type => 'string',
 612         format => 'pve-volume-id',
 613         optional => 1,
 614         description => "Script that will be executed during various steps in the vms lifetime.",
 615     },
 616     ivshmem => {
 617         type => 'string',
 618         format => $ivshmem_fmt,
 619         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 620         optional => 1,
 621     },
 622     audio0 => {
 623         type => 'string',
 624         format => $audio_fmt,
 625         description => "Configure a audio device, useful in combination with QXL/Spice.",
 626         optional => 1
 627     },
 628     spice_enhancements => {
 629         type => 'string',
 630         format => $spice_enhancements_fmt,
 631         description => "Configure additional enhancements for SPICE.",
 632         optional => 1
 633     },
 634     tags => {
 635         type => 'string', format => 'pve-tag-list',
 636         description => 'Tags of the VM. This is only meta information.',
 637         optional => 1,
 638     },
 639     rng0 => {
 640         type => 'string',
 641         format => $rng_fmt,
 642         description => "Configure a VirtIO-based Random Number Generator.",
 643         optional => 1,
 644     },
 645 };
 646
 647 my $cicustom_fmt = {
 648     meta => {
 649         type => 'string',
 650         optional => 1,
 651         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 652         format => 'pve-volume-id',
 653         format_description => 'volume',
 654     },
 655     network => {
 656         type => 'string',
 657         optional => 1,
 658         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 659         format => 'pve-volume-id',
 660         format_description => 'volume',
 661     },
 662     user => {
 663         type => 'string',
 664         optional => 1,
 665         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 666         format => 'pve-volume-id',
 667         format_description => 'volume',
 668     },
 669 };
 670 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 671
 672 my $confdesc_cloudinit = {
 673     citype => {
 674         optional => 1,
 675         type => 'string',
 676         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 677         enum => ['configdrive2', 'nocloud'],
 678     },
 679     ciuser => {
 680         optional => 1,
 681         type => 'string',
 682         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 683     },
 684     cipassword => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 688     },
 689     cicustom => {
 690         optional => 1,
 691         type => 'string',
 692         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 693         format => 'pve-qm-cicustom',
 694     },
 695     searchdomain => {
 696         optional => 1,
 697         type => 'string',
 698         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     nameserver => {
 701         optional => 1,
 702         type => 'string', format => 'address-list',
 703         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 704     },
 705     sshkeys => {
 706         optional => 1,
 707         type => 'string',
 708         format => 'urlencoded',
 709         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 710     },
 711 };
 712
 713 # what about other qemu settings ?
 714 #cpu => 'string',
 715 #machine => 'string',
 716 #fda => 'file',
 717 #fdb => 'file',
 718 #mtdblock => 'file',
 719 #sd => 'file',
 720 #pflash => 'file',
 721 #snapshot => 'bool',
 722 #bootp => 'file',
 723 ##tftp => 'dir',
 724 ##smb => 'dir',
 725 #kernel => 'file',
 726 #append => 'string',
 727 #initrd => 'file',
 728 ##soundhw => 'string',
 729
 730 while (my ($k, $v) = each %$confdesc) {
 731     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 732 }
 733
 734 my $MAX_USB_DEVICES = 5;
 735 my $MAX_NETS = 32;
 736 my $MAX_HOSTPCI_DEVICES = 16;
 737 my $MAX_SERIAL_PORTS = 4;
 738 my $MAX_PARALLEL_PORTS = 3;
 739 my $MAX_NUMA = 8;
 740
 741 my $numa_fmt = {
 742     cpus => {
 743         type => "string",
 744         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 745         description => "CPUs accessing this NUMA node.",
 746         format_description => "id[-id];...",
 747     },
 748     memory => {
 749         type => "number",
 750         description => "Amount of memory this NUMA node provides.",
 751         optional => 1,
 752     },
 753     hostnodes => {
 754         type => "string",
 755         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 756         description => "Host NUMA nodes to use.",
 757         format_description => "id[-id];...",
 758         optional => 1,
 759     },
 760     policy => {
 761         type => 'string',
 762         enum => [qw(preferred bind interleave)],
 763         description => "NUMA allocation policy.",
 764         optional => 1,
 765     },
 766 };
 767 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 768 my $numadesc = {
 769     optional => 1,
 770     type => 'string', format => $numa_fmt,
 771     description => "NUMA topology.",
 772 };
 773 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 774
 775 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 776     $confdesc->{"numa$i"} = $numadesc;
 777 }
 778
 779 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 780                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 781                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 782 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 783
 784 my $net_fmt_bridge_descr = <<__EOD__;
 785 Bridge to attach the network device to. The Proxmox VE standard bridge
 786 is called 'vmbr0'.
 787
 788 If you do not specify a bridge, we create a kvm user (NATed) network
 789 device, which provides DHCP and DNS services. The following addresses
 790 are used:
 791
 792  10.0.2.2   Gateway
 793  10.0.2.3   DNS Server
 794  10.0.2.4   SMB Server
 795
 796 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 797 __EOD__
 798
 799 my $net_fmt = {
 800     macaddr  => get_standard_option('mac-addr', {
 801         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 802     }),
 803     model => {
 804         type => 'string',
 805         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 806         enum => $nic_model_list,
 807         default_key => 1,
 808     },
 809     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 810     bridge => {
 811         type => 'string',
 812         description => $net_fmt_bridge_descr,
 813         format_description => 'bridge',
 814         optional => 1,
 815     },
 816     queues => {
 817         type => 'integer',
 818         minimum => 0, maximum => 16,
 819         description => 'Number of packet queues to be used on the device.',
 820         optional => 1,
 821     },
 822     rate => {
 823         type => 'number',
 824         minimum => 0,
 825         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 826         optional => 1,
 827     },
 828     tag => {
 829         type => 'integer',
 830         minimum => 1, maximum => 4094,
 831         description => 'VLAN tag to apply to packets on this interface.',
 832         optional => 1,
 833     },
 834     trunks => {
 835         type => 'string',
 836         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 837         description => 'VLAN trunks to pass through this interface.',
 838         format_description => 'vlanid[;vlanid...]',
 839         optional => 1,
 840     },
 841     firewall => {
 842         type => 'boolean',
 843         description => 'Whether this interface should be protected by the firewall.',
 844         optional => 1,
 845     },
 846     link_down => {
 847         type => 'boolean',
 848         description => 'Whether this interface should be disconnected (like pulling the plug).',
 849         optional => 1,
 850     },
 851 };
 852
 853 my $netdesc = {
 854     optional => 1,
 855     type => 'string', format => $net_fmt,
 856     description => "Specify network devices.",
 857 };
 858
 859 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 860
 861 my $ipconfig_fmt = {
 862     ip => {
 863         type => 'string',
 864         format => 'pve-ipv4-config',
 865         format_description => 'IPv4Format/CIDR',
 866         description => 'IPv4 address in CIDR format.',
 867         optional => 1,
 868         default => 'dhcp',
 869     },
 870     gw => {
 871         type => 'string',
 872         format => 'ipv4',
 873         format_description => 'GatewayIPv4',
 874         description => 'Default gateway for IPv4 traffic.',
 875         optional => 1,
 876         requires => 'ip',
 877     },
 878     ip6 => {
 879         type => 'string',
 880         format => 'pve-ipv6-config',
 881         format_description => 'IPv6Format/CIDR',
 882         description => 'IPv6 address in CIDR format.',
 883         optional => 1,
 884         default => 'dhcp',
 885     },
 886     gw6 => {
 887         type => 'string',
 888         format => 'ipv6',
 889         format_description => 'GatewayIPv6',
 890         description => 'Default gateway for IPv6 traffic.',
 891         optional => 1,
 892         requires => 'ip6',
 893     },
 894 };
 895 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 896 my $ipconfigdesc = {
 897     optional => 1,
 898     type => 'string', format => 'pve-qm-ipconfig',
 899     description => <<'EODESCR',
 900 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 901
 902 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 903
 904 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 905 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 906
 907 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 908 EODESCR
 909 };
 910 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 911
 912 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 913     $confdesc->{"net$i"} = $netdesc;
 914     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 915 }
 916
 917 foreach my $key (keys %$confdesc_cloudinit) {
 918     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 919 }
 920
 921 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 922 sub verify_volume_id_or_qm_path {
 923     my ($volid, $noerr) = @_;
 924
 925     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 926         return $volid;
 927     }
 928
 929     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 930     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 931     if ($@) {
 932         return undef if $noerr;
 933         die $@;
 934     }
 935     return $volid;
 936 }
 937
 938 my $usb_fmt = {
 939     host => {
 940         default_key => 1,
 941         type => 'string', format => 'pve-qm-usb-device',
 942         format_description => 'HOSTUSBDEVICE|spice',
 943         description => <<EODESCR,
 944 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 945
 946  'bus-port(.port)*' (decimal numbers) or
 947  'vendor_id:product_id' (hexadeciaml numbers) or
 948  'spice'
 949
 950 You can use the 'lsusb -t' command to list existing usb devices.
 951
 952 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 953
 954 The value 'spice' can be used to add a usb redirection devices for spice.
 955 EODESCR
 956     },
 957     usb3 => {
 958         optional => 1,
 959         type => 'boolean',
 960         description => "Specifies whether if given host option is a USB3 device or port.",
 961         default => 0,
 962     },
 963 };
 964
 965 my $usbdesc = {
 966     optional => 1,
 967     type => 'string', format => $usb_fmt,
 968     description => "Configure an USB device (n is 0 to 4).",
 969 };
 970 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 971
 972 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
 973 my $hostpci_fmt = {
 974     host => {
 975         default_key => 1,
 976         type => 'string',
 977         pattern => qr/$PCIRE(;$PCIRE)*/,
 978         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 979         description => <<EODESCR,
 980 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 981 of PCI virtual functions of the host. HOSTPCIID syntax is:
 982
 983 'bus:dev.func' (hexadecimal numbers)
 984
 985 You can us the 'lspci' command to list existing PCI devices.
 986 EODESCR
 987     },
 988     rombar => {
 989         type => 'boolean',
 990         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 991         optional => 1,
 992         default => 1,
 993     },
 994     romfile => {
 995         type => 'string',
 996         pattern => '[^,;]+',
 997         format_description => 'string',
 998         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
 999         optional => 1,
1000     },
1001     pcie => {
1002         type => 'boolean',
1003         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1004         optional => 1,
1005         default => 0,
1006     },
1007     'x-vga' => {
1008         type => 'boolean',
1009         description =>  "Enable vfio-vga device support.",
1010         optional => 1,
1011         default => 0,
1012     },
1013     'mdev' => {
1014         type => 'string',
1015         format_description => 'string',
1016         pattern => '[^/\.:]+',
1017         optional => 1,
1018         description => <<EODESCR
1019 The type of mediated device to use.
1020 An instance of this type will be created on startup of the VM and
1021 will be cleaned up when the VM stops.
1022 EODESCR
1023     }
1024 };
1025 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1026
1027 my $hostpcidesc = {
1028         optional => 1,
1029         type => 'string', format => 'pve-qm-hostpci',
1030         description => "Map host PCI devices into guest.",
1031         verbose_description =>  <<EODESCR,
1032 Map host PCI devices into guest.
1033
1034 NOTE: This option allows direct access to host hardware. So it is no longer
1035 possible to migrate such machines - use with special care.
1036
1037 CAUTION: Experimental! User reported problems with this option.
1038 EODESCR
1039 };
1040 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1041
1042 my $serialdesc = {
1043         optional => 1,
1044         type => 'string',
1045         pattern => '(/dev/.+|socket)',
1046         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1047         verbose_description =>  <<EODESCR,
1048 Create a serial device inside the VM (n is 0 to 3), and pass through a
1049 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1050 host side (use 'qm terminal' to open a terminal connection).
1051
1052 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 my $paralleldesc= {
1059         optional => 1,
1060         type => 'string',
1061         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1062         description =>  "Map host parallel devices (n is 0 to 2).",
1063         verbose_description =>  <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $PVE::QemuServer::Drive::MAX_UNUSED_DISKS; $i++) {
1089     $confdesc->{"unused$i"} = $PVE::QemuServer::Drive::unuseddesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1093     $confdesc->{"usb$i"} = $usbdesc;
1094 }
1095
1096 my $kvm_api_version = 0;
1097
1098 sub kvm_version {
1099     return $kvm_api_version if $kvm_api_version;
1100
1101     open my $fh, '<', '/dev/kvm'
1102         or return undef;
1103
1104     # 0xae00 => KVM_GET_API_VERSION
1105     $kvm_api_version = ioctl($fh, 0xae00, 0);
1106
1107     return $kvm_api_version;
1108 }
1109
1110 my $kvm_user_version = {};
1111 my $kvm_mtime = {};
1112
1113 sub kvm_user_version {
1114     my ($binary) = @_;
1115
1116     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1117     my $st = stat($binary);
1118
1119     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1120     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1121         $cachedmtime == $st->mtime;
1122
1123     $kvm_user_version->{$binary} = 'unknown';
1124     $kvm_mtime->{$binary} = $st->mtime;
1125
1126     my $code = sub {
1127         my $line = shift;
1128         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1129             $kvm_user_version->{$binary} = $2;
1130         }
1131     };
1132
1133     eval { run_command([$binary, '--version'], outfunc => $code); };
1134     warn $@ if $@;
1135
1136     return $kvm_user_version->{$binary};
1137
1138 }
1139
1140 sub kernel_has_vhost_net {
1141     return -c '/dev/vhost-net';
1142 }
1143
1144 sub option_exists {
1145     my $key = shift;
1146     return defined($confdesc->{$key});
1147 }
1148
1149 my $cdrom_path;
1150 sub get_cdrom_path {
1151
1152     return  $cdrom_path if $cdrom_path;
1153
1154     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1155     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1156     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1157 }
1158
1159 sub get_iso_path {
1160     my ($storecfg, $vmid, $cdrom) = @_;
1161
1162     if ($cdrom eq 'cdrom') {
1163         return get_cdrom_path();
1164     } elsif ($cdrom eq 'none') {
1165         return '';
1166     } elsif ($cdrom =~ m|^/|) {
1167         return $cdrom;
1168     } else {
1169         return PVE::Storage::path($storecfg, $cdrom);
1170     }
1171 }
1172
1173 # try to convert old style file names to volume IDs
1174 sub filename_to_volume_id {
1175     my ($vmid, $file, $media) = @_;
1176
1177      if (!($file eq 'none' || $file eq 'cdrom' ||
1178           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1179
1180         return undef if $file =~ m|/|;
1181
1182         if ($media && $media eq 'cdrom') {
1183             $file = "local:iso/$file";
1184         } else {
1185             $file = "local:$vmid/$file";
1186         }
1187     }
1188
1189     return $file;
1190 }
1191
1192 sub verify_media_type {
1193     my ($opt, $vtype, $media) = @_;
1194
1195     return if !$media;
1196
1197     my $etype;
1198     if ($media eq 'disk') {
1199         $etype = 'images';
1200     } elsif ($media eq 'cdrom') {
1201         $etype = 'iso';
1202     } else {
1203         die "internal error";
1204     }
1205
1206     return if ($vtype eq $etype);
1207
1208     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1209 }
1210
1211 sub cleanup_drive_path {
1212     my ($opt, $storecfg, $drive) = @_;
1213
1214     # try to convert filesystem paths to volume IDs
1215
1216     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1217         ($drive->{file} !~ m|^/dev/.+|) &&
1218         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1219         ($drive->{file} !~ m/^\d+$/)) {
1220         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1221         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1222         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1223         verify_media_type($opt, $vtype, $drive->{media});
1224         $drive->{file} = $volid;
1225     }
1226
1227     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1228 }
1229
1230 sub parse_hotplug_features {
1231     my ($data) = @_;
1232
1233     my $res = {};
1234
1235     return $res if $data eq '0';
1236
1237     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1238
1239     foreach my $feature (PVE::Tools::split_list($data)) {
1240         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1241             $res->{$1} = 1;
1242         } else {
1243             die "invalid hotplug feature '$feature'\n";
1244         }
1245     }
1246     return $res;
1247 }
1248
1249 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1250 sub pve_verify_hotplug_features {
1251     my ($value, $noerr) = @_;
1252
1253     return $value if parse_hotplug_features($value);
1254
1255     return undef if $noerr;
1256
1257     die "unable to parse hotplug option\n";
1258 }
1259
1260 sub scsi_inquiry {
1261     my($fh, $noerr) = @_;
1262
1263     my $SG_IO = 0x2285;
1264     my $SG_GET_VERSION_NUM = 0x2282;
1265
1266     my $versionbuf = "\x00" x 8;
1267     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1268     if (!$ret) {
1269         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1270         return undef;
1271     }
1272     my $version = unpack("I", $versionbuf);
1273     if ($version < 30000) {
1274         die "scsi generic interface too old\n"  if !$noerr;
1275         return undef;
1276     }
1277
1278     my $buf = "\x00" x 36;
1279     my $sensebuf = "\x00" x 8;
1280     my $cmd = pack("C x3 C x1", 0x12, 36);
1281
1282     # see /usr/include/scsi/sg.h
1283     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1284
1285     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1286                       length($sensebuf), 0, length($buf), $buf,
1287                       $cmd, $sensebuf, 6000);
1288
1289     $ret = ioctl($fh, $SG_IO, $packet);
1290     if (!$ret) {
1291         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1292         return undef;
1293     }
1294
1295     my @res = unpack($sg_io_hdr_t, $packet);
1296     if ($res[17] || $res[18]) {
1297         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1298         return undef;
1299     }
1300
1301     my $res = {};
1302     (my $byte0, my $byte1, $res->{vendor},
1303      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1304
1305     $res->{removable} = $byte1 & 128 ? 1 : 0;
1306     $res->{type} = $byte0 & 31;
1307
1308     return $res;
1309 }
1310
1311 sub path_is_scsi {
1312     my ($path) = @_;
1313
1314     my $fh = IO::File->new("+<$path") || return undef;
1315     my $res = scsi_inquiry($fh, 1);
1316     close($fh);
1317
1318     return $res;
1319 }
1320
1321 sub print_tabletdevice_full {
1322     my ($conf, $arch) = @_;
1323
1324     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1325
1326     # we use uhci for old VMs because tablet driver was buggy in older qemu
1327     my $usbbus;
1328     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1329         $usbbus = 'ehci';
1330     } else {
1331         $usbbus = 'uhci';
1332     }
1333
1334     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1335 }
1336
1337 sub print_keyboarddevice_full {
1338     my ($conf, $arch, $machine) = @_;
1339
1340     return undef if $arch ne 'aarch64';
1341
1342     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1343 }
1344
1345 sub print_drivedevice_full {
1346     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1347
1348     my $device = '';
1349     my $maxdev = 0;
1350
1351     if ($drive->{interface} eq 'virtio') {
1352         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1353         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1354         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1355     } elsif ($drive->{interface} eq 'scsi') {
1356
1357         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1358         my $unit = $drive->{index} % $maxdev;
1359         my $devicetype = 'hd';
1360         my $path = '';
1361         if (drive_is_cdrom($drive)) {
1362             $devicetype = 'cd';
1363         } else {
1364             if ($drive->{file} =~ m|^/|) {
1365                 $path = $drive->{file};
1366                 if (my $info = path_is_scsi($path)) {
1367                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1368                         $devicetype = 'block';
1369                     } elsif ($info->{type} == 1) { # tape
1370                         $devicetype = 'generic';
1371                     }
1372                 }
1373             } else {
1374                  $path = PVE::Storage::path($storecfg, $drive->{file});
1375             }
1376
1377             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1378             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1379             if ($path =~ m/^iscsi\:\/\// &&
1380                !min_version($version, 4, 1)) {
1381                 $devicetype = 'generic';
1382             }
1383         }
1384
1385         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1386             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1387         } else {
1388             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1389         }
1390
1391         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1392             $device .= ",rotation_rate=1";
1393         }
1394         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1395
1396     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1397         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1398         my $controller = int($drive->{index} / $maxdev);
1399         my $unit = $drive->{index} % $maxdev;
1400         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1401
1402         $device = "ide-$devicetype";
1403         if ($drive->{interface} eq 'ide') {
1404             $device .= ",bus=ide.$controller,unit=$unit";
1405         } else {
1406             $device .= ",bus=ahci$controller.$unit";
1407         }
1408         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1409
1410         if ($devicetype eq 'hd') {
1411             if (my $model = $drive->{model}) {
1412                 $model = URI::Escape::uri_unescape($model);
1413                 $device .= ",model=$model";
1414             }
1415             if ($drive->{ssd}) {
1416                 $device .= ",rotation_rate=1";
1417             }
1418         }
1419         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1420     } elsif ($drive->{interface} eq 'usb') {
1421         die "implement me";
1422         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1423     } else {
1424         die "unsupported interface type";
1425     }
1426
1427     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1428
1429     if (my $serial = $drive->{serial}) {
1430         $serial = URI::Escape::uri_unescape($serial);
1431         $device .= ",serial=$serial";
1432     }
1433
1434
1435     return $device;
1436 }
1437
1438 sub get_initiator_name {
1439     my $initiator;
1440
1441     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1442     while (defined(my $line = <$fh>)) {
1443         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1444         $initiator = $1;
1445         last;
1446     }
1447     $fh->close();
1448
1449     return $initiator;
1450 }
1451
1452 sub print_drive_commandline_full {
1453     my ($storecfg, $vmid, $drive) = @_;
1454
1455     my $path;
1456     my $volid = $drive->{file};
1457     my $format;
1458
1459     if (drive_is_cdrom($drive)) {
1460         $path = get_iso_path($storecfg, $vmid, $volid);
1461     } else {
1462         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1463         if ($storeid) {
1464             $path = PVE::Storage::path($storecfg, $volid);
1465             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1466             $format = qemu_img_format($scfg, $volname);
1467         } else {
1468             $path = $volid;
1469             $format = "raw";
1470         }
1471    }
1472
1473     my $opts = '';
1474     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1475     foreach my $o (@qemu_drive_options) {
1476         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1477     }
1478
1479     # snapshot only accepts on|off
1480     if (defined($drive->{snapshot})) {
1481         my $v = $drive->{snapshot} ? 'on' : 'off';
1482         $opts .= ",snapshot=$v";
1483     }
1484
1485     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1486         my ($dir, $qmpname) = @$type;
1487         if (my $v = $drive->{"mbps$dir"}) {
1488             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1489         }
1490         if (my $v = $drive->{"mbps${dir}_max"}) {
1491             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1492         }
1493         if (my $v = $drive->{"bps${dir}_max_length"}) {
1494             $opts .= ",throttling.bps$qmpname-max-length=$v";
1495         }
1496         if (my $v = $drive->{"iops${dir}"}) {
1497             $opts .= ",throttling.iops$qmpname=$v";
1498         }
1499         if (my $v = $drive->{"iops${dir}_max"}) {
1500             $opts .= ",throttling.iops$qmpname-max=$v";
1501         }
1502         if (my $v = $drive->{"iops${dir}_max_length"}) {
1503             $opts .= ",throttling.iops$qmpname-max-length=$v";
1504         }
1505     }
1506
1507     $opts .= ",format=$format" if $format && !$drive->{format};
1508
1509     my $cache_direct = 0;
1510
1511     if (my $cache = $drive->{cache}) {
1512         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1513     } elsif (!drive_is_cdrom($drive)) {
1514         $opts .= ",cache=none";
1515         $cache_direct = 1;
1516     }
1517
1518     # aio native works only with O_DIRECT
1519     if (!$drive->{aio}) {
1520         if($cache_direct) {
1521             $opts .= ",aio=native";
1522         } else {
1523             $opts .= ",aio=threads";
1524         }
1525     }
1526
1527     if (!drive_is_cdrom($drive)) {
1528         my $detectzeroes;
1529         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1530             $detectzeroes = 'off';
1531         } elsif ($drive->{discard}) {
1532             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1533         } else {
1534             # This used to be our default with discard not being specified:
1535             $detectzeroes = 'on';
1536         }
1537         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1538     }
1539
1540     my $pathinfo = $path ? "file=$path," : '';
1541
1542     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1543 }
1544
1545 sub print_netdevice_full {
1546     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1547
1548     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1549
1550     my $device = $net->{model};
1551     if ($net->{model} eq 'virtio') {
1552          $device = 'virtio-net-pci';
1553      };
1554
1555     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1556     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1557     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1558         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1559         my $vectors = $net->{queues} * 2 + 2;
1560         $tmpstr .= ",vectors=$vectors,mq=on";
1561     }
1562     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1563
1564     if ($use_old_bios_files) {
1565         my $romfile;
1566         if ($device eq 'virtio-net-pci') {
1567             $romfile = 'pxe-virtio.rom';
1568         } elsif ($device eq 'e1000') {
1569             $romfile = 'pxe-e1000.rom';
1570         } elsif ($device eq 'ne2k') {
1571             $romfile = 'pxe-ne2k_pci.rom';
1572         } elsif ($device eq 'pcnet') {
1573             $romfile = 'pxe-pcnet.rom';
1574         } elsif ($device eq 'rtl8139') {
1575             $romfile = 'pxe-rtl8139.rom';
1576         }
1577         $tmpstr .= ",romfile=$romfile" if $romfile;
1578     }
1579
1580     return $tmpstr;
1581 }
1582
1583 sub print_netdev_full {
1584     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1585
1586     my $i = '';
1587     if ($netid =~ m/^net(\d+)$/) {
1588         $i = int($1);
1589     }
1590
1591     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1592
1593     my $ifname = "tap${vmid}i$i";
1594
1595     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1596     die "interface name '$ifname' is too long (max 15 character)\n"
1597         if length($ifname) >= 16;
1598
1599     my $vhostparam = '';
1600     if (is_native($arch)) {
1601         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1602     }
1603
1604     my $vmname = $conf->{name} || "vm$vmid";
1605
1606     my $netdev = "";
1607     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1608
1609     if ($net->{bridge}) {
1610         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1611     } else {
1612         $netdev = "type=user,id=$netid,hostname=$vmname";
1613     }
1614
1615     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1616
1617     return $netdev;
1618 }
1619
1620 my $vga_map = {
1621     'cirrus' => 'cirrus-vga',
1622     'std' => 'VGA',
1623     'vmware' => 'vmware-svga',
1624     'virtio' => 'virtio-vga',
1625 };
1626
1627 sub print_vga_device {
1628     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1629
1630     my $type = $vga_map->{$vga->{type}};
1631     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1632         $type = 'virtio-gpu';
1633     }
1634     my $vgamem_mb = $vga->{memory};
1635
1636     my $max_outputs = '';
1637     if ($qxlnum) {
1638         $type = $id ? 'qxl' : 'qxl-vga';
1639
1640         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1641             # set max outputs so linux can have up to 4 qxl displays with one device
1642             if (min_version($machine_version, 4, 1)) {
1643                 $max_outputs = ",max_outputs=4";
1644             }
1645         }
1646     }
1647
1648     die "no devicetype for $vga->{type}\n" if !$type;
1649
1650     my $memory = "";
1651     if ($vgamem_mb) {
1652         if ($vga->{type} eq 'virtio') {
1653             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1654             $memory = ",max_hostmem=$bytes";
1655         } elsif ($qxlnum) {
1656             # from https://www.spice-space.org/multiple-monitors.html
1657             $memory = ",vgamem_mb=$vga->{memory}";
1658             my $ram = $vgamem_mb * 4;
1659             my $vram = $vgamem_mb * 2;
1660             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1661         } else {
1662             $memory = ",vgamem_mb=$vga->{memory}";
1663         }
1664     } elsif ($qxlnum && $id) {
1665         $memory = ",ram_size=67108864,vram_size=33554432";
1666     }
1667
1668     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1669     my $vgaid = "vga" . ($id // '');
1670     my $pciaddr;
1671
1672     if ($q35 && $vgaid eq 'vga') {
1673         # the first display uses pcie.0 bus on q35 machines
1674         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1675     } else {
1676         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1677     }
1678
1679     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1680 }
1681
1682 sub parse_number_sets {
1683     my ($set) = @_;
1684     my $res = [];
1685     foreach my $part (split(/;/, $set)) {
1686         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1687             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1688             push @$res, [ $1, $2 ];
1689         } else {
1690             die "invalid range: $part\n";
1691         }
1692     }
1693     return $res;
1694 }
1695
1696 sub parse_numa {
1697     my ($data) = @_;
1698
1699     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1700     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1701     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1702     return $res;
1703 }
1704
1705 sub parse_hostpci {
1706     my ($value) = @_;
1707
1708     return undef if !$value;
1709
1710     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1711
1712     my @idlist = split(/;/, $res->{host});
1713     delete $res->{host};
1714     foreach my $id (@idlist) {
1715         my $devs = PVE::SysFSTools::lspci($id);
1716         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1717         push @{$res->{pciid}}, @$devs;
1718     }
1719     return $res;
1720 }
1721
1722 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1723 sub parse_net {
1724     my ($data) = @_;
1725
1726     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1727     if ($@) {
1728         warn $@;
1729         return undef;
1730     }
1731     if (!defined($res->{macaddr})) {
1732         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1733         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1734     }
1735     return $res;
1736 }
1737
1738 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1739 sub parse_ipconfig {
1740     my ($data) = @_;
1741
1742     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1743     if ($@) {
1744         warn $@;
1745         return undef;
1746     }
1747
1748     if ($res->{gw} && !$res->{ip}) {
1749         warn 'gateway specified without specifying an IP address';
1750         return undef;
1751     }
1752     if ($res->{gw6} && !$res->{ip6}) {
1753         warn 'IPv6 gateway specified without specifying an IPv6 address';
1754         return undef;
1755     }
1756     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1757         warn 'gateway specified together with DHCP';
1758         return undef;
1759     }
1760     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1761         # gw6 + auto/dhcp
1762         warn "IPv6 gateway specified together with $res->{ip6} address";
1763         return undef;
1764     }
1765
1766     if (!$res->{ip} && !$res->{ip6}) {
1767         return { ip => 'dhcp', ip6 => 'dhcp' };
1768     }
1769
1770     return $res;
1771 }
1772
1773 sub print_net {
1774     my $net = shift;
1775
1776     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1777 }
1778
1779 sub add_random_macs {
1780     my ($settings) = @_;
1781
1782     foreach my $opt (keys %$settings) {
1783         next if $opt !~ m/^net(\d+)$/;
1784         my $net = parse_net($settings->{$opt});
1785         next if !$net;
1786         $settings->{$opt} = print_net($net);
1787     }
1788 }
1789
1790 sub vm_is_volid_owner {
1791     my ($storecfg, $vmid, $volid) = @_;
1792
1793     if ($volid !~  m|^/|) {
1794         my ($path, $owner);
1795         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1796         if ($owner && ($owner == $vmid)) {
1797             return 1;
1798         }
1799     }
1800
1801     return undef;
1802 }
1803
1804 sub vmconfig_register_unused_drive {
1805     my ($storecfg, $vmid, $conf, $drive) = @_;
1806
1807     if (drive_is_cloudinit($drive)) {
1808         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1809         warn $@ if $@;
1810     } elsif (!drive_is_cdrom($drive)) {
1811         my $volid = $drive->{file};
1812         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1813             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1814         }
1815     }
1816 }
1817
1818 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1819 my $smbios1_fmt = {
1820     uuid => {
1821         type => 'string',
1822         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1823         format_description => 'UUID',
1824         description => "Set SMBIOS1 UUID.",
1825         optional => 1,
1826     },
1827     version => {
1828         type => 'string',
1829         pattern => '[A-Za-z0-9+\/]+={0,2}',
1830         format_description => 'Base64 encoded string',
1831         description => "Set SMBIOS1 version.",
1832         optional => 1,
1833     },
1834     serial => {
1835         type => 'string',
1836         pattern => '[A-Za-z0-9+\/]+={0,2}',
1837         format_description => 'Base64 encoded string',
1838         description => "Set SMBIOS1 serial number.",
1839         optional => 1,
1840     },
1841     manufacturer => {
1842         type => 'string',
1843         pattern => '[A-Za-z0-9+\/]+={0,2}',
1844         format_description => 'Base64 encoded string',
1845         description => "Set SMBIOS1 manufacturer.",
1846         optional => 1,
1847     },
1848     product => {
1849         type => 'string',
1850         pattern => '[A-Za-z0-9+\/]+={0,2}',
1851         format_description => 'Base64 encoded string',
1852         description => "Set SMBIOS1 product ID.",
1853         optional => 1,
1854     },
1855     sku => {
1856         type => 'string',
1857         pattern => '[A-Za-z0-9+\/]+={0,2}',
1858         format_description => 'Base64 encoded string',
1859         description => "Set SMBIOS1 SKU string.",
1860         optional => 1,
1861     },
1862     family => {
1863         type => 'string',
1864         pattern => '[A-Za-z0-9+\/]+={0,2}',
1865         format_description => 'Base64 encoded string',
1866         description => "Set SMBIOS1 family string.",
1867         optional => 1,
1868     },
1869     base64 => {
1870         type => 'boolean',
1871         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1872         optional => 1,
1873     },
1874 };
1875
1876 sub parse_smbios1 {
1877     my ($data) = @_;
1878
1879     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1880     warn $@ if $@;
1881     return $res;
1882 }
1883
1884 sub print_smbios1 {
1885     my ($smbios1) = @_;
1886     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1887 }
1888
1889 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1890
1891 sub parse_watchdog {
1892     my ($value) = @_;
1893
1894     return undef if !$value;
1895
1896     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1897     warn $@ if $@;
1898     return $res;
1899 }
1900
1901 sub parse_guest_agent {
1902     my ($value) = @_;
1903
1904     return {} if !defined($value->{agent});
1905
1906     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1907     warn $@ if $@;
1908
1909     # if the agent is disabled ignore the other potentially set properties
1910     return {} if !$res->{enabled};
1911     return $res;
1912 }
1913
1914 sub parse_vga {
1915     my ($value) = @_;
1916
1917     return {} if !$value;
1918     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1919     warn $@ if $@;
1920     return $res;
1921 }
1922
1923 sub parse_rng {
1924     my ($value) = @_;
1925
1926     return undef if !$value;
1927
1928     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1929     warn $@ if $@;
1930     return $res;
1931 }
1932
1933 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1934 sub verify_usb_device {
1935     my ($value, $noerr) = @_;
1936
1937     return $value if parse_usb_device($value);
1938
1939     return undef if $noerr;
1940
1941     die "unable to parse usb device\n";
1942 }
1943
1944 # add JSON properties for create and set function
1945 sub json_config_properties {
1946     my $prop = shift;
1947
1948     foreach my $opt (keys %$confdesc) {
1949         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1950         $prop->{$opt} = $confdesc->{$opt};
1951     }
1952
1953     return $prop;
1954 }
1955
1956 # return copy of $confdesc_cloudinit to generate documentation
1957 sub cloudinit_config_properties {
1958
1959     return dclone($confdesc_cloudinit);
1960 }
1961
1962 sub check_type {
1963     my ($key, $value) = @_;
1964
1965     die "unknown setting '$key'\n" if !$confdesc->{$key};
1966
1967     my $type = $confdesc->{$key}->{type};
1968
1969     if (!defined($value)) {
1970         die "got undefined value\n";
1971     }
1972
1973     if ($value =~ m/[\n\r]/) {
1974         die "property contains a line feed\n";
1975     }
1976
1977     if ($type eq 'boolean') {
1978         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1979         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1980         die "type check ('boolean') failed - got '$value'\n";
1981     } elsif ($type eq 'integer') {
1982         return int($1) if $value =~ m/^(\d+)$/;
1983         die "type check ('integer') failed - got '$value'\n";
1984     } elsif ($type eq 'number') {
1985         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1986         die "type check ('number') failed - got '$value'\n";
1987     } elsif ($type eq 'string') {
1988         if (my $fmt = $confdesc->{$key}->{format}) {
1989             PVE::JSONSchema::check_format($fmt, $value);
1990             return $value;
1991         }
1992         $value =~ s/^\"(.*)\"$/$1/;
1993         return $value;
1994     } else {
1995         die "internal error"
1996     }
1997 }
1998
1999 sub destroy_vm {
2000     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2001
2002     my $conf = PVE::QemuConfig->load_config($vmid);
2003
2004     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2005
2006     if ($conf->{template}) {
2007         # check if any base image is still used by a linked clone
2008         foreach_drive($conf, sub {
2009                 my ($ds, $drive) = @_;
2010                 return if drive_is_cdrom($drive);
2011
2012                 my $volid = $drive->{file};
2013                 return if !$volid || $volid =~ m|^/|;
2014
2015                 die "base volume '$volid' is still in use by linked cloned\n"
2016                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2017
2018         });
2019     }
2020
2021     # only remove disks owned by this VM
2022     foreach_drive($conf, sub {
2023         my ($ds, $drive) = @_;
2024         return if drive_is_cdrom($drive, 1);
2025
2026         my $volid = $drive->{file};
2027         return if !$volid || $volid =~ m|^/|;
2028
2029         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2030         return if !$path || !$owner || ($owner != $vmid);
2031
2032         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2033         warn "Could not remove disk '$volid', check manually: $@" if $@;
2034     });
2035
2036     # also remove unused disk
2037     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2038     PVE::Storage::foreach_volid($vmdisks, sub {
2039         my ($volid, $sid, $volname, $d) = @_;
2040         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2041         warn $@ if $@;
2042     });
2043
2044     if (defined $replacement_conf) {
2045         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2046     } else {
2047         PVE::QemuConfig->destroy_config($vmid);
2048     }
2049 }
2050
2051 sub parse_vm_config {
2052     my ($filename, $raw) = @_;
2053
2054     return undef if !defined($raw);
2055
2056     my $res = {
2057         digest => Digest::SHA::sha1_hex($raw),
2058         snapshots => {},
2059         pending => {},
2060     };
2061
2062     $filename =~ m|/qemu-server/(\d+)\.conf$|
2063         || die "got strange filename '$filename'";
2064
2065     my $vmid = $1;
2066
2067     my $conf = $res;
2068     my $descr;
2069     my $section = '';
2070
2071     my @lines = split(/\n/, $raw);
2072     foreach my $line (@lines) {
2073         next if $line =~ m/^\s*$/;
2074
2075         if ($line =~ m/^\[PENDING\]\s*$/i) {
2076             $section = 'pending';
2077             if (defined($descr)) {
2078                 $descr =~ s/\s+$//;
2079                 $conf->{description} = $descr;
2080             }
2081             $descr = undef;
2082             $conf = $res->{$section} = {};
2083             next;
2084
2085         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2086             $section = $1;
2087             if (defined($descr)) {
2088                 $descr =~ s/\s+$//;
2089                 $conf->{description} = $descr;
2090             }
2091             $descr = undef;
2092             $conf = $res->{snapshots}->{$section} = {};
2093             next;
2094         }
2095
2096         if ($line =~ m/^\#(.*)\s*$/) {
2097             $descr = '' if !defined($descr);
2098             $descr .= PVE::Tools::decode_text($1) . "\n";
2099             next;
2100         }
2101
2102         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2103             $descr = '' if !defined($descr);
2104             $descr .= PVE::Tools::decode_text($2);
2105         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2106             $conf->{snapstate} = $1;
2107         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2108             my $key = $1;
2109             my $value = $2;
2110             $conf->{$key} = $value;
2111         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2112             my $value = $1;
2113             if ($section eq 'pending') {
2114                 $conf->{delete} = $value; # we parse this later
2115             } else {
2116                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2117             }
2118         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2119             my $key = $1;
2120             my $value = $2;
2121             eval { $value = check_type($key, $value); };
2122             if ($@) {
2123                 warn "vm $vmid - unable to parse value of '$key' - $@";
2124             } else {
2125                 $key = 'ide2' if $key eq 'cdrom';
2126                 my $fmt = $confdesc->{$key}->{format};
2127                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2128                     my $v = parse_drive($key, $value);
2129                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2130                         $v->{file} = $volid;
2131                         $value = print_drive($v);
2132                     } else {
2133                         warn "vm $vmid - unable to parse value of '$key'\n";
2134                         next;
2135                     }
2136                 }
2137
2138                 $conf->{$key} = $value;
2139             }
2140         }
2141     }
2142
2143     if (defined($descr)) {
2144         $descr =~ s/\s+$//;
2145         $conf->{description} = $descr;
2146     }
2147     delete $res->{snapstate}; # just to be sure
2148
2149     return $res;
2150 }
2151
2152 sub write_vm_config {
2153     my ($filename, $conf) = @_;
2154
2155     delete $conf->{snapstate}; # just to be sure
2156
2157     if ($conf->{cdrom}) {
2158         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2159         $conf->{ide2} = $conf->{cdrom};
2160         delete $conf->{cdrom};
2161     }
2162
2163     # we do not use 'smp' any longer
2164     if ($conf->{sockets}) {
2165         delete $conf->{smp};
2166     } elsif ($conf->{smp}) {
2167         $conf->{sockets} = $conf->{smp};
2168         delete $conf->{cores};
2169         delete $conf->{smp};
2170     }
2171
2172     my $used_volids = {};
2173
2174     my $cleanup_config = sub {
2175         my ($cref, $pending, $snapname) = @_;
2176
2177         foreach my $key (keys %$cref) {
2178             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2179                 $key eq 'snapstate' || $key eq 'pending';
2180             my $value = $cref->{$key};
2181             if ($key eq 'delete') {
2182                 die "propertry 'delete' is only allowed in [PENDING]\n"
2183                     if !$pending;
2184                 # fixme: check syntax?
2185                 next;
2186             }
2187             eval { $value = check_type($key, $value); };
2188             die "unable to parse value of '$key' - $@" if $@;
2189
2190             $cref->{$key} = $value;
2191
2192             if (!$snapname && is_valid_drivename($key)) {
2193                 my $drive = parse_drive($key, $value);
2194                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2195             }
2196         }
2197     };
2198
2199     &$cleanup_config($conf);
2200
2201     &$cleanup_config($conf->{pending}, 1);
2202
2203     foreach my $snapname (keys %{$conf->{snapshots}}) {
2204         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2205         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2206     }
2207
2208     # remove 'unusedX' settings if we re-add a volume
2209     foreach my $key (keys %$conf) {
2210         my $value = $conf->{$key};
2211         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2212             delete $conf->{$key};
2213         }
2214     }
2215
2216     my $generate_raw_config = sub {
2217         my ($conf, $pending) = @_;
2218
2219         my $raw = '';
2220
2221         # add description as comment to top of file
2222         if (defined(my $descr = $conf->{description})) {
2223             if ($descr) {
2224                 foreach my $cl (split(/\n/, $descr)) {
2225                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2226                 }
2227             } else {
2228                 $raw .= "#\n" if $pending;
2229             }
2230         }
2231
2232         foreach my $key (sort keys %$conf) {
2233             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2234             $raw .= "$key: $conf->{$key}\n";
2235         }
2236         return $raw;
2237     };
2238
2239     my $raw = &$generate_raw_config($conf);
2240
2241     if (scalar(keys %{$conf->{pending}})){
2242         $raw .= "\n[PENDING]\n";
2243         $raw .= &$generate_raw_config($conf->{pending}, 1);
2244     }
2245
2246     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2247         $raw .= "\n[$snapname]\n";
2248         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2249     }
2250
2251     return $raw;
2252 }
2253
2254 sub load_defaults {
2255
2256     my $res = {};
2257
2258     # we use static defaults from our JSON schema configuration
2259     foreach my $key (keys %$confdesc) {
2260         if (defined(my $default = $confdesc->{$key}->{default})) {
2261             $res->{$key} = $default;
2262         }
2263     }
2264
2265     return $res;
2266 }
2267
2268 sub config_list {
2269     my $vmlist = PVE::Cluster::get_vmlist();
2270     my $res = {};
2271     return $res if !$vmlist || !$vmlist->{ids};
2272     my $ids = $vmlist->{ids};
2273     my $nodename = nodename();
2274
2275     foreach my $vmid (keys %$ids) {
2276         my $d = $ids->{$vmid};
2277         next if !$d->{node} || $d->{node} ne $nodename;
2278         next if !$d->{type} || $d->{type} ne 'qemu';
2279         $res->{$vmid}->{exists} = 1;
2280     }
2281     return $res;
2282 }
2283
2284 # test if VM uses local resources (to prevent migration)
2285 sub check_local_resources {
2286     my ($conf, $noerr) = @_;
2287
2288     my @loc_res = ();
2289
2290     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2291     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2292
2293     push @loc_res, "ivshmem" if $conf->{ivshmem};
2294
2295     foreach my $k (keys %$conf) {
2296         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2297         # sockets are safe: they will recreated be on the target side post-migrate
2298         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2299         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2300     }
2301
2302     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2303
2304     return \@loc_res;
2305 }
2306
2307 # check if used storages are available on all nodes (use by migrate)
2308 sub check_storage_availability {
2309     my ($storecfg, $conf, $node) = @_;
2310
2311     foreach_drive($conf, sub {
2312         my ($ds, $drive) = @_;
2313
2314         my $volid = $drive->{file};
2315         return if !$volid;
2316
2317         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2318         return if !$sid;
2319
2320         # check if storage is available on both nodes
2321         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2322         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2323    });
2324 }
2325
2326 # list nodes where all VM images are available (used by has_feature API)
2327 sub shared_nodes {
2328     my ($conf, $storecfg) = @_;
2329
2330     my $nodelist = PVE::Cluster::get_nodelist();
2331     my $nodehash = { map { $_ => 1 } @$nodelist };
2332     my $nodename = nodename();
2333
2334     foreach_drive($conf, sub {
2335         my ($ds, $drive) = @_;
2336
2337         my $volid = $drive->{file};
2338         return if !$volid;
2339
2340         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2341         if ($storeid) {
2342             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2343             if ($scfg->{disable}) {
2344                 $nodehash = {};
2345             } elsif (my $avail = $scfg->{nodes}) {
2346                 foreach my $node (keys %$nodehash) {
2347                     delete $nodehash->{$node} if !$avail->{$node};
2348                 }
2349             } elsif (!$scfg->{shared}) {
2350                 foreach my $node (keys %$nodehash) {
2351                     delete $nodehash->{$node} if $node ne $nodename
2352                 }
2353             }
2354         }
2355     });
2356
2357     return $nodehash
2358 }
2359
2360 sub check_local_storage_availability {
2361     my ($conf, $storecfg) = @_;
2362
2363     my $nodelist = PVE::Cluster::get_nodelist();
2364     my $nodehash = { map { $_ => {} } @$nodelist };
2365
2366     foreach_drive($conf, sub {
2367         my ($ds, $drive) = @_;
2368
2369         my $volid = $drive->{file};
2370         return if !$volid;
2371
2372         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2373         if ($storeid) {
2374             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2375
2376             if ($scfg->{disable}) {
2377                 foreach my $node (keys %$nodehash) {
2378                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2379                 }
2380             } elsif (my $avail = $scfg->{nodes}) {
2381                 foreach my $node (keys %$nodehash) {
2382                     if (!$avail->{$node}) {
2383                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2384                     }
2385                 }
2386             }
2387         }
2388     });
2389
2390     foreach my $node (values %$nodehash) {
2391         if (my $unavail = $node->{unavailable_storages}) {
2392             $node->{unavailable_storages} = [ sort keys %$unavail ];
2393         }
2394     }
2395
2396     return $nodehash
2397 }
2398
2399 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2400 sub check_running {
2401     my ($vmid, $nocheck, $node) = @_;
2402
2403     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2404     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2405 }
2406
2407 sub vzlist {
2408
2409     my $vzlist = config_list();
2410
2411     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2412
2413     while (defined(my $de = $fd->read)) {
2414         next if $de !~ m/^(\d+)\.pid$/;
2415         my $vmid = $1;
2416         next if !defined($vzlist->{$vmid});
2417         if (my $pid = check_running($vmid)) {
2418             $vzlist->{$vmid}->{pid} = $pid;
2419         }
2420     }
2421
2422     return $vzlist;
2423 }
2424
2425 our $vmstatus_return_properties = {
2426     vmid => get_standard_option('pve-vmid'),
2427     status => {
2428         description => "Qemu process status.",
2429         type => 'string',
2430         enum => ['stopped', 'running'],
2431     },
2432     maxmem => {
2433         description => "Maximum memory in bytes.",
2434         type => 'integer',
2435         optional => 1,
2436         renderer => 'bytes',
2437     },
2438     maxdisk => {
2439         description => "Root disk size in bytes.",
2440         type => 'integer',
2441         optional => 1,
2442         renderer => 'bytes',
2443     },
2444     name => {
2445         description => "VM name.",
2446         type => 'string',
2447         optional => 1,
2448     },
2449     qmpstatus => {
2450         description => "Qemu QMP agent status.",
2451         type => 'string',
2452         optional => 1,
2453     },
2454     pid => {
2455         description => "PID of running qemu process.",
2456         type => 'integer',
2457         optional => 1,
2458     },
2459     uptime => {
2460         description => "Uptime.",
2461         type => 'integer',
2462         optional => 1,
2463         renderer => 'duration',
2464     },
2465     cpus => {
2466         description => "Maximum usable CPUs.",
2467         type => 'number',
2468         optional => 1,
2469     },
2470     lock => {
2471         description => "The current config lock, if any.",
2472         type => 'string',
2473         optional => 1,
2474     },
2475     tags => {
2476         description  => "The current configured tags, if any",
2477         type => 'string',
2478         optional => 1,
2479     },
2480 };
2481
2482 my $last_proc_pid_stat;
2483
2484 # get VM status information
2485 # This must be fast and should not block ($full == false)
2486 # We only query KVM using QMP if $full == true (this can be slow)
2487 sub vmstatus {
2488     my ($opt_vmid, $full) = @_;
2489
2490     my $res = {};
2491
2492     my $storecfg = PVE::Storage::config();
2493
2494     my $list = vzlist();
2495     my $defaults = load_defaults();
2496
2497     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2498
2499     my $cpucount = $cpuinfo->{cpus} || 1;
2500
2501     foreach my $vmid (keys %$list) {
2502         next if $opt_vmid && ($vmid ne $opt_vmid);
2503
2504         my $conf = PVE::QemuConfig->load_config($vmid);
2505
2506         my $d = { vmid => $vmid };
2507         $d->{pid} = $list->{$vmid}->{pid};
2508
2509         # fixme: better status?
2510         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2511
2512         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2513         if (defined($size)) {
2514             $d->{disk} = 0; # no info available
2515             $d->{maxdisk} = $size;
2516         } else {
2517             $d->{disk} = 0;
2518             $d->{maxdisk} = 0;
2519         }
2520
2521         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2522             * ($conf->{cores} || $defaults->{cores});
2523         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2524         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2525
2526         $d->{name} = $conf->{name} || "VM $vmid";
2527         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2528             : $defaults->{memory}*(1024*1024);
2529
2530         if ($conf->{balloon}) {
2531             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2532             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2533                 : $defaults->{shares};
2534         }
2535
2536         $d->{uptime} = 0;
2537         $d->{cpu} = 0;
2538         $d->{mem} = 0;
2539
2540         $d->{netout} = 0;
2541         $d->{netin} = 0;
2542
2543         $d->{diskread} = 0;
2544         $d->{diskwrite} = 0;
2545
2546         $d->{template} = PVE::QemuConfig->is_template($conf);
2547
2548         $d->{serial} = 1 if conf_has_serial($conf);
2549         $d->{lock} = $conf->{lock} if $conf->{lock};
2550         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2551
2552         $res->{$vmid} = $d;
2553     }
2554
2555     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2556     foreach my $dev (keys %$netdev) {
2557         next if $dev !~ m/^tap([1-9]\d*)i/;
2558         my $vmid = $1;
2559         my $d = $res->{$vmid};
2560         next if !$d;
2561
2562         $d->{netout} += $netdev->{$dev}->{receive};
2563         $d->{netin} += $netdev->{$dev}->{transmit};
2564
2565         if ($full) {
2566             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2567             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2568         }
2569
2570     }
2571
2572     my $ctime = gettimeofday;
2573
2574     foreach my $vmid (keys %$list) {
2575
2576         my $d = $res->{$vmid};
2577         my $pid = $d->{pid};
2578         next if !$pid;
2579
2580         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2581         next if !$pstat; # not running
2582
2583         my $used = $pstat->{utime} + $pstat->{stime};
2584
2585         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2586
2587         if ($pstat->{vsize}) {
2588             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2589         }
2590
2591         my $old = $last_proc_pid_stat->{$pid};
2592         if (!$old) {
2593             $last_proc_pid_stat->{$pid} = {
2594                 time => $ctime,
2595                 used => $used,
2596                 cpu => 0,
2597             };
2598             next;
2599         }
2600
2601         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2602
2603         if ($dtime > 1000) {
2604             my $dutime = $used -  $old->{used};
2605
2606             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2607             $last_proc_pid_stat->{$pid} = {
2608                 time => $ctime,
2609                 used => $used,
2610                 cpu => $d->{cpu},
2611             };
2612         } else {
2613             $d->{cpu} = $old->{cpu};
2614         }
2615     }
2616
2617     return $res if !$full;
2618
2619     my $qmpclient = PVE::QMPClient->new();
2620
2621     my $ballooncb = sub {
2622         my ($vmid, $resp) = @_;
2623
2624         my $info = $resp->{'return'};
2625         return if !$info->{max_mem};
2626
2627         my $d = $res->{$vmid};
2628
2629         # use memory assigned to VM
2630         $d->{maxmem} = $info->{max_mem};
2631         $d->{balloon} = $info->{actual};
2632
2633         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2634             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2635             $d->{freemem} = $info->{free_mem};
2636         }
2637
2638         $d->{ballooninfo} = $info;
2639     };
2640
2641     my $blockstatscb = sub {
2642         my ($vmid, $resp) = @_;
2643         my $data = $resp->{'return'} || [];
2644         my $totalrdbytes = 0;
2645         my $totalwrbytes = 0;
2646
2647         for my $blockstat (@$data) {
2648             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2649             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2650
2651             $blockstat->{device} =~ s/drive-//;
2652             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2653         }
2654         $res->{$vmid}->{diskread} = $totalrdbytes;
2655         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2656     };
2657
2658     my $statuscb = sub {
2659         my ($vmid, $resp) = @_;
2660
2661         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2662         # this fails if ballon driver is not loaded, so this must be
2663         # the last commnand (following command are aborted if this fails).
2664         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2665
2666         my $status = 'unknown';
2667         if (!defined($status = $resp->{'return'}->{status})) {
2668             warn "unable to get VM status\n";
2669             return;
2670         }
2671
2672         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2673     };
2674
2675     foreach my $vmid (keys %$list) {
2676         next if $opt_vmid && ($vmid ne $opt_vmid);
2677         next if !$res->{$vmid}->{pid}; # not running
2678         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2679     }
2680
2681     $qmpclient->queue_execute(undef, 2);
2682
2683     foreach my $vmid (keys %$list) {
2684         next if $opt_vmid && ($vmid ne $opt_vmid);
2685         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2686     }
2687
2688     return $res;
2689 }
2690
2691 sub conf_has_serial {
2692     my ($conf) = @_;
2693
2694     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2695         if ($conf->{"serial$i"}) {
2696             return 1;
2697         }
2698     }
2699
2700     return 0;
2701 }
2702
2703 sub conf_has_audio {
2704     my ($conf, $id) = @_;
2705
2706     $id //= 0;
2707     my $audio = $conf->{"audio$id"};
2708     return undef if !defined($audio);
2709
2710     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2711     my $audiodriver = $audioproperties->{driver} // 'spice';
2712
2713     return {
2714         dev => $audioproperties->{device},
2715         dev_id => "audiodev$id",
2716         backend => $audiodriver,
2717         backend_id => "$audiodriver-backend${id}",
2718     };
2719 }
2720
2721 sub vga_conf_has_spice {
2722     my ($vga) = @_;
2723
2724     my $vgaconf = parse_vga($vga);
2725     my $vgatype = $vgaconf->{type};
2726     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2727
2728     return $1 || 1;
2729 }
2730
2731 sub is_native($) {
2732     my ($arch) = @_;
2733     return get_host_arch() eq $arch;
2734 }
2735
2736 sub get_vm_arch {
2737     my ($conf) = @_;
2738     return $conf->{arch} // get_host_arch();
2739 }
2740
2741 my $default_machines = {
2742     x86_64 => 'pc',
2743     aarch64 => 'virt',
2744 };
2745
2746 sub get_vm_machine {
2747     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2748
2749     my $machine = $forcemachine || $conf->{machine};
2750
2751     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2752         $arch //= 'x86_64';
2753         $machine ||= $default_machines->{$arch};
2754         if ($add_pve_version) {
2755             $kvmversion //= kvm_user_version();
2756             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2757             $machine .= "+pve$pvever";
2758         }
2759     }
2760
2761     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2762         # for version-pinned machines that do not include a pve-version (e.g.
2763         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2764         $machine .= '+pve0';
2765     }
2766
2767     return $machine;
2768 }
2769
2770 sub get_ovmf_files($) {
2771     my ($arch) = @_;
2772
2773     my $ovmf = $OVMF->{$arch}
2774         or die "no OVMF images known for architecture '$arch'\n";
2775
2776     return @$ovmf;
2777 }
2778
2779 my $Arch2Qemu = {
2780     aarch64 => '/usr/bin/qemu-system-aarch64',
2781     x86_64 => '/usr/bin/qemu-system-x86_64',
2782 };
2783 sub get_command_for_arch($) {
2784     my ($arch) = @_;
2785     return '/usr/bin/kvm' if is_native($arch);
2786
2787     my $cmd = $Arch2Qemu->{$arch}
2788         or die "don't know how to emulate architecture '$arch'\n";
2789     return $cmd;
2790 }
2791
2792 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2793 # to use in a QEMU command line (-cpu element), first array_intersect the result
2794 # of query_supported_ with query_understood_. This is necessary because:
2795 #
2796 # a) query_understood_ returns flags the host cannot use and
2797 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2798 #    flags, but CPU settings - with most of them being flags. Those settings
2799 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2800 #
2801 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2802 # expensive. If you need the value returned from this, you can get it much
2803 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2804 # $accel being 'kvm' or 'tcg'.
2805 #
2806 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2807 # changes, automatically populating pmxcfs.
2808 #
2809 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2810 # since kvm and tcg machines support different flags
2811 #
2812 sub query_supported_cpu_flags {
2813     my ($arch) = @_;
2814
2815     $arch //= get_host_arch();
2816     my $default_machine = $default_machines->{$arch};
2817
2818     my $flags = {};
2819
2820     # FIXME: Once this is merged, the code below should work for ARM as well:
2821     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2822     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2823         $arch eq "aarch64";
2824
2825     my $kvm_supported = defined(kvm_version());
2826     my $qemu_cmd = get_command_for_arch($arch);
2827     my $fakevmid = -1;
2828     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2829
2830     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2831     my $query_supported_run_qemu = sub {
2832         my ($kvm) = @_;
2833
2834         my $flags = {};
2835         my $cmd = [
2836             $qemu_cmd,
2837             '-machine', $default_machine,
2838             '-display', 'none',
2839             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2840             '-mon', 'chardev=qmp,mode=control',
2841             '-pidfile', $pidfile,
2842             '-S', '-daemonize'
2843         ];
2844
2845         if (!$kvm) {
2846             push @$cmd, '-accel', 'tcg';
2847         }
2848
2849         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2850         die "QEMU flag querying VM exited with code " . $rc if $rc;
2851
2852         eval {
2853             my $cmd_result = mon_cmd(
2854                 $fakevmid,
2855                 'query-cpu-model-expansion',
2856                 type => 'full',
2857                 model => { name => 'host' }
2858             );
2859
2860             my $props = $cmd_result->{model}->{props};
2861             foreach my $prop (keys %$props) {
2862                 next if $props->{$prop} ne '1';
2863                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2864                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2865                 # We only keep those with underscores, to match /proc/cpuinfo
2866                 $prop =~ s/\.|-/_/g;
2867                 $flags->{$prop} = 1;
2868             }
2869         };
2870         my $err = $@;
2871
2872         # force stop with 10 sec timeout and 'nocheck'
2873         # always stop, even if QMP failed
2874         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2875
2876         die $err if $err;
2877
2878         return [ sort keys %$flags ];
2879     };
2880
2881     # We need to query QEMU twice, since KVM and TCG have different supported flags
2882     PVE::QemuConfig->lock_config($fakevmid, sub {
2883         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2884         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2885
2886         if ($kvm_supported) {
2887             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2888             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2889         }
2890     });
2891
2892     return $flags;
2893 }
2894
2895 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2896 my $understood_cpu_flag_dir = "/usr/share/kvm";
2897 sub query_understood_cpu_flags {
2898     my $arch = get_host_arch();
2899     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2900
2901     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2902         if ! -e $filepath;
2903
2904     my $raw = file_get_contents($filepath);
2905     $raw =~ s/^\s+|\s+$//g;
2906     my @flags = split(/\s+/, $raw);
2907
2908     return \@flags;
2909 }
2910
2911 sub config_to_command {
2912     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2913
2914     my $cmd = [];
2915     my $globalFlags = [];
2916     my $machineFlags = [];
2917     my $rtcFlags = [];
2918     my $devices = [];
2919     my $pciaddr = '';
2920     my $bridges = {};
2921     my $ostype = $conf->{ostype};
2922     my $winversion = windows_version($ostype);
2923     my $kvm = $conf->{kvm};
2924     my $nodename = nodename();
2925
2926     my $arch = get_vm_arch($conf);
2927     my $kvm_binary = get_command_for_arch($arch);
2928     my $kvmver = kvm_user_version($kvm_binary);
2929
2930     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2931         $kvmver //= "undefined";
2932         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2933     }
2934
2935     my $add_pve_version = min_version($kvmver, 4, 1);
2936
2937     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2938     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2939     $kvm //= 1 if is_native($arch);
2940
2941     $machine_version =~ m/(\d+)\.(\d+)/;
2942     my ($machine_major, $machine_minor) = ($1, $2);
2943     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2944         if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
2945
2946     if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2947         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2948         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2949           . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2950     }
2951
2952     # if a specific +pve version is required for a feature, use $version_guard
2953     # instead of min_version to allow machines to be run with the minimum
2954     # required version
2955     my $required_pve_version = 0;
2956     my $version_guard = sub {
2957         my ($major, $minor, $pve) = @_;
2958         return 0 if !min_version($machine_version, $major, $minor, $pve);
2959         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2960         return 1;
2961     };
2962
2963     if ($kvm) {
2964         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2965             if !defined kvm_version();
2966     }
2967
2968     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2969     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2970     my $use_old_bios_files = undef;
2971     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2972
2973     my $cpuunits = defined($conf->{cpuunits}) ?
2974             $conf->{cpuunits} : $defaults->{cpuunits};
2975
2976     push @$cmd, $kvm_binary;
2977
2978     push @$cmd, '-id', $vmid;
2979
2980     my $vmname = $conf->{name} || "vm$vmid";
2981
2982     push @$cmd, '-name', $vmname;
2983
2984     my $use_virtio = 0;
2985
2986     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2987     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2988     push @$cmd, '-mon', "chardev=qmp,mode=control";
2989
2990     if (min_version($machine_version, 2, 12)) {
2991         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2992         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2993     }
2994
2995     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
2996
2997     push @$cmd, '-daemonize';
2998
2999     if ($conf->{smbios1}) {
3000         my $smbios_conf = parse_smbios1($conf->{smbios1});
3001         if ($smbios_conf->{base64}) {
3002             # Do not pass base64 flag to qemu
3003             delete $smbios_conf->{base64};
3004             my $smbios_string = "";
3005             foreach my $key (keys %$smbios_conf) {
3006                 my $value;
3007                 if ($key eq "uuid") {
3008                     $value = $smbios_conf->{uuid}
3009                 } else {
3010                     $value = decode_base64($smbios_conf->{$key});
3011                 }
3012                 # qemu accepts any binary data, only commas need escaping by double comma
3013                 $value =~ s/,/,,/g;
3014                 $smbios_string .= "," . $key . "=" . $value if $value;
3015             }
3016             push @$cmd, '-smbios', "type=1" . $smbios_string;
3017         } else {
3018             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3019         }
3020     }
3021
3022     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3023     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3024         die "uefi base image not found\n" if ! -f $ovmf_code;
3025
3026         my $path;
3027         my $format;
3028         if (my $efidisk = $conf->{efidisk0}) {
3029             my $d = parse_drive('efidisk0', $efidisk);
3030             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3031             $format = $d->{format};
3032             if ($storeid) {
3033                 $path = PVE::Storage::path($storecfg, $d->{file});
3034                 if (!defined($format)) {
3035                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3036                     $format = qemu_img_format($scfg, $volname);
3037                 }
3038             } else {
3039                 $path = $d->{file};
3040                 die "efidisk format must be specified\n"
3041                     if !defined($format);
3042             }
3043         } else {
3044             warn "no efidisk configured! Using temporary efivars disk.\n";
3045             $path = "/tmp/$vmid-ovmf.fd";
3046             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3047             $format = 'raw';
3048         }
3049
3050         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3051         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3052     }
3053
3054     # load q35 config
3055     if ($q35) {
3056         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3057         if (min_version($machine_version, 4, 0)) {
3058             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3059         } else {
3060             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3061         }
3062     }
3063
3064     if ($conf->{vmgenid}) {
3065         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3066     }
3067
3068     # add usb controllers
3069     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3070     push @$devices, @usbcontrollers if @usbcontrollers;
3071     my $vga = parse_vga($conf->{vga});
3072
3073     my $qxlnum = vga_conf_has_spice($conf->{vga});
3074     $vga->{type} = 'qxl' if $qxlnum;
3075
3076     if (!$vga->{type}) {
3077         if ($arch eq 'aarch64') {
3078             $vga->{type} = 'virtio';
3079         } elsif (min_version($machine_version, 2, 9)) {
3080             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3081         } else {
3082             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3083         }
3084     }
3085
3086     # enable absolute mouse coordinates (needed by vnc)
3087     my $tablet;
3088     if (defined($conf->{tablet})) {
3089         $tablet = $conf->{tablet};
3090     } else {
3091         $tablet = $defaults->{tablet};
3092         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3093         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3094     }
3095
3096     if ($tablet) {
3097         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3098         my $kbd = print_keyboarddevice_full($conf, $arch);
3099         push @$devices, '-device', $kbd if defined($kbd);
3100     }
3101
3102     my $kvm_off = 0;
3103     my $gpu_passthrough;
3104
3105     # host pci devices
3106     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3107         my $id = "hostpci$i";
3108         my $d = parse_hostpci($conf->{$id});
3109         next if !$d;
3110
3111         if (my $pcie = $d->{pcie}) {
3112             die "q35 machine model is not enabled" if !$q35;
3113             # win7 wants to have the pcie devices directly on the pcie bus
3114             # instead of in the root port
3115             if ($winversion == 7) {
3116                 $pciaddr = print_pcie_addr("${id}bus0");
3117             } else {
3118                 # add more root ports if needed, 4 are present by default
3119                 # by pve-q35 cfgs, rest added here on demand.
3120                 if ($i > 3) {
3121                     push @$devices, '-device', print_pcie_root_port($i);
3122                 }
3123                 $pciaddr = print_pcie_addr($id);
3124             }
3125         } else {
3126             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3127         }
3128
3129         my $xvga = '';
3130         if ($d->{'x-vga'}) {
3131             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3132             $kvm_off = 1;
3133             $vga->{type} = 'none' if !defined($conf->{vga});
3134             $gpu_passthrough = 1;
3135         }
3136
3137         my $pcidevices = $d->{pciid};
3138         my $multifunction = 1 if @$pcidevices > 1;
3139
3140         my $sysfspath;
3141         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3142             my $pci_id = $pcidevices->[0]->{id};
3143             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3144             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3145         } elsif ($d->{mdev}) {
3146             warn "ignoring mediated device '$id' with multifunction device\n";
3147         }
3148
3149         my $j=0;
3150         foreach my $pcidevice (@$pcidevices) {
3151             my $devicestr = "vfio-pci";
3152
3153             if ($sysfspath) {
3154                 $devicestr .= ",sysfsdev=$sysfspath";
3155             } else {
3156                 $devicestr .= ",host=$pcidevice->{id}";
3157             }
3158
3159             my $mf_addr = $multifunction ? ".$j" : '';
3160             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3161
3162             if ($j == 0) {
3163                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3164                 $devicestr .= "$xvga";
3165                 $devicestr .= ",multifunction=on" if $multifunction;
3166                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3167             }
3168
3169             push @$devices, '-device', $devicestr;
3170             $j++;
3171         }
3172     }
3173
3174     # usb devices
3175     my $usb_dev_features = {};
3176     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3177
3178     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3179     push @$devices, @usbdevices if @usbdevices;
3180     # serial devices
3181     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3182         if (my $path = $conf->{"serial$i"}) {
3183             if ($path eq 'socket') {
3184                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3185                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3186                 # On aarch64, serial0 is the UART device. Qemu only allows
3187                 # connecting UART devices via the '-serial' command line, as
3188                 # the device has a fixed slot on the hardware...
3189                 if ($arch eq 'aarch64' && $i == 0) {
3190                     push @$devices, '-serial', "chardev:serial$i";
3191                 } else {
3192                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3193                 }
3194             } else {
3195                 die "no such serial device\n" if ! -c $path;
3196                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3197                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3198             }
3199         }
3200     }
3201
3202     # parallel devices
3203     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3204         if (my $path = $conf->{"parallel$i"}) {
3205             die "no such parallel device\n" if ! -c $path;
3206             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3207             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3208             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3209         }
3210     }
3211
3212     if (my $audio = conf_has_audio($conf)) {
3213
3214         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3215
3216         my $id = $audio->{dev_id};
3217         if ($audio->{dev} eq 'AC97') {
3218             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3219         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3220             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3221             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3222             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3223         } else {
3224             die "unkown audio device '$audio->{dev}', implement me!";
3225         }
3226
3227         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3228     }
3229
3230     my $sockets = 1;
3231     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3232     $sockets = $conf->{sockets} if  $conf->{sockets};
3233
3234     my $cores = $conf->{cores} || 1;
3235
3236     my $maxcpus = $sockets * $cores;
3237
3238     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3239
3240     my $allowed_vcpus = $cpuinfo->{cpus};
3241
3242     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3243         if ($allowed_vcpus < $maxcpus);
3244
3245     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3246
3247         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3248         for (my $i = 2; $i <= $vcpus; $i++)  {
3249             my $cpustr = print_cpu_device($conf,$i);
3250             push @$cmd, '-device', $cpustr;
3251         }
3252
3253     } else {
3254
3255         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3256     }
3257     push @$cmd, '-nodefaults';
3258
3259     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3260
3261     my $bootindex_hash = {};
3262     my $i = 1;
3263     foreach my $o (split(//, $bootorder)) {
3264         $bootindex_hash->{$o} = $i*100;
3265         $i++;
3266     }
3267
3268     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3269
3270     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3271
3272     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3273
3274     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3275         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3276         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3277         push @$cmd,  '-vnc', "unix:$socket,password";
3278     } else {
3279         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3280         push @$cmd, '-nographic';
3281     }
3282
3283     # time drift fix
3284     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3285
3286     my $useLocaltime = $conf->{localtime};
3287
3288     if ($winversion >= 5) { # windows
3289         $useLocaltime = 1 if !defined($conf->{localtime});
3290
3291         # use time drift fix when acpi is enabled
3292         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3293             $tdf = 1 if !defined($conf->{tdf});
3294         }
3295     }
3296
3297     if ($winversion >= 6) {
3298         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3299         push @$cmd, '-no-hpet';
3300     }
3301
3302     push @$rtcFlags, 'driftfix=slew' if $tdf;
3303
3304     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3305         push @$rtcFlags, "base=$conf->{startdate}";
3306     } elsif ($useLocaltime) {
3307         push @$rtcFlags, 'base=localtime';
3308     }
3309
3310     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3311
3312     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3313
3314     push @$cmd, '-S' if $conf->{freeze};
3315
3316     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3317
3318     my $guest_agent = parse_guest_agent($conf);
3319
3320     if ($guest_agent->{enabled}) {
3321         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3322         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3323
3324         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3325             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3326             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3327             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3328         } elsif ($guest_agent->{type} eq 'isa') {
3329             push @$devices, '-device', "isa-serial,chardev=qga0";
3330         }
3331     }
3332
3333     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3334     if ($rng && &$version_guard(4, 1, 2)) {
3335         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3336         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3337
3338         my $limiter_str = "";
3339         if ($max_bytes) {
3340             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3341         }
3342
3343         # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3344         die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3345             if ! -e $rng->{source};
3346
3347         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3348
3349         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3350         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3351     }
3352
3353     my $spice_port;
3354
3355     if ($qxlnum) {
3356         if ($qxlnum > 1) {
3357             if ($winversion){
3358                 for(my $i = 1; $i < $qxlnum; $i++){
3359                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3360                 }
3361             } else {
3362                 # assume other OS works like Linux
3363                 my ($ram, $vram) = ("134217728", "67108864");
3364                 if ($vga->{memory}) {
3365                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3366                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3367                 }
3368                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3369                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3370             }
3371         }
3372
3373         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3374
3375         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3376         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3377         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3378
3379         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3380         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3381         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3382
3383         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3384         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3385
3386         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3387         if ($spice_enhancement->{foldersharing}) {
3388             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3389             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3390         }
3391
3392         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3393         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3394         push @$devices, '-spice', "$spice_opts";
3395     }
3396
3397     # enable balloon by default, unless explicitly disabled
3398     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3399         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3400         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3401     }
3402
3403     if ($conf->{watchdog}) {
3404         my $wdopts = parse_watchdog($conf->{watchdog});
3405         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3406         my $watchdog = $wdopts->{model} || 'i6300esb';
3407         push @$devices, '-device', "$watchdog$pciaddr";
3408         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3409     }
3410
3411     my $vollist = [];
3412     my $scsicontroller = {};
3413     my $ahcicontroller = {};
3414     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3415
3416     # Add iscsi initiator name if available
3417     if (my $initiator = get_initiator_name()) {
3418         push @$devices, '-iscsi', "initiator-name=$initiator";
3419     }
3420
3421     foreach_drive($conf, sub {
3422         my ($ds, $drive) = @_;
3423
3424         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3425             push @$vollist, $drive->{file};
3426         }
3427
3428         # ignore efidisk here, already added in bios/fw handling code above
3429         return if $drive->{interface} eq 'efidisk';
3430
3431         $use_virtio = 1 if $ds =~ m/^virtio/;
3432
3433         if (drive_is_cdrom ($drive)) {
3434             if ($bootindex_hash->{d}) {
3435                 $drive->{bootindex} = $bootindex_hash->{d};
3436                 $bootindex_hash->{d} += 1;
3437             }
3438         } else {
3439             if ($bootindex_hash->{c}) {
3440                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3441                 $bootindex_hash->{c} += 1;
3442             }
3443         }
3444
3445         if($drive->{interface} eq 'virtio'){
3446            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3447         }
3448
3449         if ($drive->{interface} eq 'scsi') {
3450
3451             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3452
3453             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3454                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3455
3456             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3457             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3458
3459             my $iothread = '';
3460             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3461                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3462                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3463             } elsif ($drive->{iothread}) {
3464                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3465             }
3466
3467             my $queues = '';
3468             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3469                 $queues = ",num_queues=$drive->{queues}";
3470             }
3471
3472             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3473             $scsicontroller->{$controller}=1;
3474         }
3475
3476         if ($drive->{interface} eq 'sata') {
3477            my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3478            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3479            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3480            $ahcicontroller->{$controller}=1;
3481         }
3482
3483         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3484         push @$devices, '-drive',$drive_cmd;
3485         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3486     });
3487
3488     for (my $i = 0; $i < $MAX_NETS; $i++) {
3489          next if !$conf->{"net$i"};
3490          my $d = parse_net($conf->{"net$i"});
3491          next if !$d;
3492
3493          $use_virtio = 1 if $d->{model} eq 'virtio';
3494
3495          if ($bootindex_hash->{n}) {
3496             $d->{bootindex} = $bootindex_hash->{n};
3497             $bootindex_hash->{n} += 1;
3498          }
3499
3500          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3501          push @$devices, '-netdev', $netdevfull;
3502
3503          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3504          push @$devices, '-device', $netdevicefull;
3505     }
3506
3507     if ($conf->{ivshmem}) {
3508         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3509
3510         my $bus;
3511         if ($q35) {
3512             $bus = print_pcie_addr("ivshmem");
3513         } else {
3514             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3515         }
3516
3517         my $ivshmem_name = $ivshmem->{name} // $vmid;
3518         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3519
3520         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3521         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3522     }
3523
3524     # pci.4 is nested in pci.1
3525     $bridges->{1} = 1 if $bridges->{4};
3526
3527     if (!$q35) {
3528         # add pci bridges
3529         if (min_version($machine_version, 2, 3)) {
3530            $bridges->{1} = 1;
3531            $bridges->{2} = 1;
3532         }
3533
3534         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3535
3536     }
3537
3538     for my $k (sort {$b cmp $a} keys %$bridges) {
3539         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3540         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3541         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3542         if ($q35) {
3543             # add after -readconfig pve-q35.cfg
3544             splice @$devices, 2, 0, '-device', $devstr;
3545         } else {
3546             unshift @$devices, '-device', $devstr if $k > 0;
3547         }
3548     }
3549
3550     if (!$kvm) {
3551         push @$machineFlags, 'accel=tcg';
3552     }
3553
3554     my $machine_type_min = $machine_type;
3555     if ($add_pve_version) {
3556         $machine_type_min =~ s/\+pve\d+$//;
3557         $machine_type_min .= "+pve$required_pve_version";
3558     }
3559     push @$machineFlags, "type=${machine_type_min}";
3560
3561     push @$cmd, @$devices;
3562     push @$cmd, '-rtc', join(',', @$rtcFlags)
3563         if scalar(@$rtcFlags);
3564     push @$cmd, '-machine', join(',', @$machineFlags)
3565         if scalar(@$machineFlags);
3566     push @$cmd, '-global', join(',', @$globalFlags)
3567         if scalar(@$globalFlags);
3568
3569     if (my $vmstate = $conf->{vmstate}) {
3570         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3571         push @$vollist, $vmstate;
3572         push @$cmd, '-loadstate', $statepath;
3573         print "activating and using '$vmstate' as vmstate\n";
3574     }
3575
3576     # add custom args
3577     if ($conf->{args}) {
3578         my $aa = PVE::Tools::split_args($conf->{args});
3579         push @$cmd, @$aa;
3580     }
3581
3582     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3583 }
3584
3585 sub spice_port {
3586     my ($vmid) = @_;
3587
3588     my $res = mon_cmd($vmid, 'query-spice');
3589
3590     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3591 }
3592
3593 sub vm_devices_list {
3594     my ($vmid) = @_;
3595
3596     my $res = mon_cmd($vmid, 'query-pci');
3597     my $devices_to_check = [];
3598     my $devices = {};
3599     foreach my $pcibus (@$res) {
3600         push @$devices_to_check, @{$pcibus->{devices}},
3601     }
3602
3603     while (@$devices_to_check) {
3604         my $to_check = [];
3605         for my $d (@$devices_to_check) {
3606             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3607             next if !$d->{'pci_bridge'};
3608
3609             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3610             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3611         }
3612         $devices_to_check = $to_check;
3613     }
3614
3615     my $resblock = mon_cmd($vmid, 'query-block');
3616     foreach my $block (@$resblock) {
3617         if($block->{device} =~ m/^drive-(\S+)/){
3618                 $devices->{$1} = 1;
3619         }
3620     }
3621
3622     my $resmice = mon_cmd($vmid, 'query-mice');
3623     foreach my $mice (@$resmice) {
3624         if ($mice->{name} eq 'QEMU HID Tablet') {
3625             $devices->{tablet} = 1;
3626             last;
3627         }
3628     }
3629
3630     # for usb devices there is no query-usb
3631     # but we can iterate over the entries in
3632     # qom-list path=/machine/peripheral
3633     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3634     foreach my $per (@$resperipheral) {
3635         if ($per->{name} =~ m/^usb\d+$/) {
3636             $devices->{$per->{name}} = 1;
3637         }
3638     }
3639
3640     return $devices;
3641 }
3642
3643 sub vm_deviceplug {
3644     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3645
3646     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3647
3648     my $devices_list = vm_devices_list($vmid);
3649     return 1 if defined($devices_list->{$deviceid});
3650
3651     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3652
3653     if ($deviceid eq 'tablet') {
3654
3655         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3656
3657     } elsif ($deviceid eq 'keyboard') {
3658
3659         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3660
3661     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3662
3663         die "usb hotplug currently not reliable\n";
3664         # since we can't reliably hot unplug all added usb devices
3665         # and usb passthrough disables live migration
3666         # we disable usb hotplugging for now
3667         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3668
3669     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3670
3671         qemu_iothread_add($vmid, $deviceid, $device);
3672
3673         qemu_driveadd($storecfg, $vmid, $device);
3674         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3675
3676         qemu_deviceadd($vmid, $devicefull);
3677         eval { qemu_deviceaddverify($vmid, $deviceid); };
3678         if (my $err = $@) {
3679             eval { qemu_drivedel($vmid, $deviceid); };
3680             warn $@ if $@;
3681             die $err;
3682         }
3683
3684     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3685
3686
3687         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3688         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3689         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3690
3691         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3692
3693         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3694             qemu_iothread_add($vmid, $deviceid, $device);
3695             $devicefull .= ",iothread=iothread-$deviceid";
3696         }
3697
3698         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3699             $devicefull .= ",num_queues=$device->{queues}";
3700         }
3701
3702         qemu_deviceadd($vmid, $devicefull);
3703         qemu_deviceaddverify($vmid, $deviceid);
3704
3705     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3706
3707         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3708         qemu_driveadd($storecfg, $vmid, $device);
3709
3710         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3711         eval { qemu_deviceadd($vmid, $devicefull); };
3712         if (my $err = $@) {
3713             eval { qemu_drivedel($vmid, $deviceid); };
3714             warn $@ if $@;
3715             die $err;
3716         }
3717
3718     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3719
3720         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3721
3722         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3723         my $use_old_bios_files = undef;
3724         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3725
3726         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3727         qemu_deviceadd($vmid, $netdevicefull);
3728         eval {
3729             qemu_deviceaddverify($vmid, $deviceid);
3730             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3731         };
3732         if (my $err = $@) {
3733             eval { qemu_netdevdel($vmid, $deviceid); };
3734             warn $@ if $@;
3735             die $err;
3736         }
3737
3738     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3739
3740         my $bridgeid = $2;
3741         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3742         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3743
3744         qemu_deviceadd($vmid, $devicefull);
3745         qemu_deviceaddverify($vmid, $deviceid);
3746
3747     } else {
3748         die "can't hotplug device '$deviceid'\n";
3749     }
3750
3751     return 1;
3752 }
3753
3754 # fixme: this should raise exceptions on error!
3755 sub vm_deviceunplug {
3756     my ($vmid, $conf, $deviceid) = @_;
3757
3758     my $devices_list = vm_devices_list($vmid);
3759     return 1 if !defined($devices_list->{$deviceid});
3760
3761     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3762
3763     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3764
3765         qemu_devicedel($vmid, $deviceid);
3766
3767     } elsif ($deviceid =~ m/^usb\d+$/) {
3768
3769         die "usb hotplug currently not reliable\n";
3770         # when unplugging usb devices this way,
3771         # there may be remaining usb controllers/hubs
3772         # so we disable it for now
3773         qemu_devicedel($vmid, $deviceid);
3774         qemu_devicedelverify($vmid, $deviceid);
3775
3776     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3777
3778         qemu_devicedel($vmid, $deviceid);
3779         qemu_devicedelverify($vmid, $deviceid);
3780         qemu_drivedel($vmid, $deviceid);
3781         qemu_iothread_del($conf, $vmid, $deviceid);
3782
3783     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3784
3785         qemu_devicedel($vmid, $deviceid);
3786         qemu_devicedelverify($vmid, $deviceid);
3787         qemu_iothread_del($conf, $vmid, $deviceid);
3788
3789     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3790
3791         qemu_devicedel($vmid, $deviceid);
3792         qemu_drivedel($vmid, $deviceid);
3793         qemu_deletescsihw($conf, $vmid, $deviceid);
3794
3795     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3796
3797         qemu_devicedel($vmid, $deviceid);
3798         qemu_devicedelverify($vmid, $deviceid);
3799         qemu_netdevdel($vmid, $deviceid);
3800
3801     } else {
3802         die "can't unplug device '$deviceid'\n";
3803     }
3804
3805     return 1;
3806 }
3807
3808 sub qemu_deviceadd {
3809     my ($vmid, $devicefull) = @_;
3810
3811     $devicefull = "driver=".$devicefull;
3812     my %options =  split(/[=,]/, $devicefull);
3813
3814     mon_cmd($vmid, "device_add" , %options);
3815 }
3816
3817 sub qemu_devicedel {
3818     my ($vmid, $deviceid) = @_;
3819
3820     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3821 }
3822
3823 sub qemu_iothread_add {
3824     my($vmid, $deviceid, $device) = @_;
3825
3826     if ($device->{iothread}) {
3827         my $iothreads = vm_iothreads_list($vmid);
3828         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3829     }
3830 }
3831
3832 sub qemu_iothread_del {
3833     my($conf, $vmid, $deviceid) = @_;
3834
3835     my $confid = $deviceid;
3836     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3837         $confid = 'scsi' . $1;
3838     }
3839     my $device = parse_drive($confid, $conf->{$confid});
3840     if ($device->{iothread}) {
3841         my $iothreads = vm_iothreads_list($vmid);
3842         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3843     }
3844 }
3845
3846 sub qemu_objectadd {
3847     my($vmid, $objectid, $qomtype) = @_;
3848
3849     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3850
3851     return 1;
3852 }
3853
3854 sub qemu_objectdel {
3855     my($vmid, $objectid) = @_;
3856
3857     mon_cmd($vmid, "object-del", id => $objectid);
3858
3859     return 1;
3860 }
3861
3862 sub qemu_driveadd {
3863     my ($storecfg, $vmid, $device) = @_;
3864
3865     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3866     $drive =~ s/\\/\\\\/g;
3867     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3868
3869     # If the command succeeds qemu prints: "OK"
3870     return 1 if $ret =~ m/OK/s;
3871
3872     die "adding drive failed: $ret\n";
3873 }
3874
3875 sub qemu_drivedel {
3876     my($vmid, $deviceid) = @_;
3877
3878     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3879     $ret =~ s/^\s+//;
3880
3881     return 1 if $ret eq "";
3882
3883     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3884     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3885
3886     die "deleting drive $deviceid failed : $ret\n";
3887 }
3888
3889 sub qemu_deviceaddverify {
3890     my ($vmid, $deviceid) = @_;
3891
3892     for (my $i = 0; $i <= 5; $i++) {
3893          my $devices_list = vm_devices_list($vmid);
3894          return 1 if defined($devices_list->{$deviceid});
3895          sleep 1;
3896     }
3897
3898     die "error on hotplug device '$deviceid'\n";
3899 }
3900
3901
3902 sub qemu_devicedelverify {
3903     my ($vmid, $deviceid) = @_;
3904
3905     # need to verify that the device is correctly removed as device_del
3906     # is async and empty return is not reliable
3907
3908     for (my $i = 0; $i <= 5; $i++) {
3909          my $devices_list = vm_devices_list($vmid);
3910          return 1 if !defined($devices_list->{$deviceid});
3911          sleep 1;
3912     }
3913
3914     die "error on hot-unplugging device '$deviceid'\n";
3915 }
3916
3917 sub qemu_findorcreatescsihw {
3918     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3919
3920     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3921
3922     my $scsihwid="$controller_prefix$controller";
3923     my $devices_list = vm_devices_list($vmid);
3924
3925     if(!defined($devices_list->{$scsihwid})) {
3926         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3927     }
3928
3929     return 1;
3930 }
3931
3932 sub qemu_deletescsihw {
3933     my ($conf, $vmid, $opt) = @_;
3934
3935     my $device = parse_drive($opt, $conf->{$opt});
3936
3937     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3938         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3939         return 1;
3940     }
3941
3942     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3943
3944     my $devices_list = vm_devices_list($vmid);
3945     foreach my $opt (keys %{$devices_list}) {
3946         if (is_valid_drivename($opt)) {
3947             my $drive = parse_drive($opt, $conf->{$opt});
3948             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3949                 return 1;
3950             }
3951         }
3952     }
3953
3954     my $scsihwid="scsihw$controller";
3955
3956     vm_deviceunplug($vmid, $conf, $scsihwid);
3957
3958     return 1;
3959 }
3960
3961 sub qemu_add_pci_bridge {
3962     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3963
3964     my $bridges = {};
3965
3966     my $bridgeid;
3967
3968     print_pci_addr($device, $bridges, $arch, $machine_type);
3969
3970     while (my ($k, $v) = each %$bridges) {
3971         $bridgeid = $k;
3972     }
3973     return 1 if !defined($bridgeid) || $bridgeid < 1;
3974
3975     my $bridge = "pci.$bridgeid";
3976     my $devices_list = vm_devices_list($vmid);
3977
3978     if (!defined($devices_list->{$bridge})) {
3979         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3980     }
3981
3982     return 1;
3983 }
3984
3985 sub qemu_set_link_status {
3986     my ($vmid, $device, $up) = @_;
3987
3988     mon_cmd($vmid, "set_link", name => $device,
3989                up => $up ? JSON::true : JSON::false);
3990 }
3991
3992 sub qemu_netdevadd {
3993     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3994
3995     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3996     my %options =  split(/[=,]/, $netdev);
3997
3998     mon_cmd($vmid, "netdev_add",  %options);
3999     return 1;
4000 }
4001
4002 sub qemu_netdevdel {
4003     my ($vmid, $deviceid) = @_;
4004
4005     mon_cmd($vmid, "netdev_del", id => $deviceid);
4006 }
4007
4008 sub qemu_usb_hotplug {
4009     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4010
4011     return if !$device;
4012
4013     # remove the old one first
4014     vm_deviceunplug($vmid, $conf, $deviceid);
4015
4016     # check if xhci controller is necessary and available
4017     if ($device->{usb3}) {
4018
4019         my $devicelist = vm_devices_list($vmid);
4020
4021         if (!$devicelist->{xhci}) {
4022             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4023             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4024         }
4025     }
4026     my $d = parse_usb_device($device->{host});
4027     $d->{usb3} = $device->{usb3};
4028
4029     # add the new one
4030     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4031 }
4032
4033 sub qemu_cpu_hotplug {
4034     my ($vmid, $conf, $vcpus) = @_;
4035
4036     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4037
4038     my $sockets = 1;
4039     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4040     $sockets = $conf->{sockets} if  $conf->{sockets};
4041     my $cores = $conf->{cores} || 1;
4042     my $maxcpus = $sockets * $cores;
4043
4044     $vcpus = $maxcpus if !$vcpus;
4045
4046     die "you can't add more vcpus than maxcpus\n"
4047         if $vcpus > $maxcpus;
4048
4049     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4050
4051     if ($vcpus < $currentvcpus) {
4052
4053         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4054
4055             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4056                 qemu_devicedel($vmid, "cpu$i");
4057                 my $retry = 0;
4058                 my $currentrunningvcpus = undef;
4059                 while (1) {
4060                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4061                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4062                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4063                     $retry++;
4064                     sleep 1;
4065                 }
4066                 #update conf after each succesfull cpu unplug
4067                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4068                 PVE::QemuConfig->write_config($vmid, $conf);
4069             }
4070         } else {
4071             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4072         }
4073
4074         return;
4075     }
4076
4077     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4078     die "vcpus in running vm does not match its configuration\n"
4079         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4080
4081     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4082
4083         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4084             my $cpustr = print_cpu_device($conf, $i);
4085             qemu_deviceadd($vmid, $cpustr);
4086
4087             my $retry = 0;
4088             my $currentrunningvcpus = undef;
4089             while (1) {
4090                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4091                 last if scalar(@{$currentrunningvcpus}) == $i;
4092                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4093                 sleep 1;
4094                 $retry++;
4095             }
4096             #update conf after each succesfull cpu hotplug
4097             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4098             PVE::QemuConfig->write_config($vmid, $conf);
4099         }
4100     } else {
4101
4102         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4103             mon_cmd($vmid, "cpu-add", id => int($i));
4104         }
4105     }
4106 }
4107
4108 sub qemu_block_set_io_throttle {
4109     my ($vmid, $deviceid,
4110         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4111         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4112         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4113         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4114
4115     return if !check_running($vmid) ;
4116
4117     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4118         bps => int($bps),
4119         bps_rd => int($bps_rd),
4120         bps_wr => int($bps_wr),
4121         iops => int($iops),
4122         iops_rd => int($iops_rd),
4123         iops_wr => int($iops_wr),
4124         bps_max => int($bps_max),
4125         bps_rd_max => int($bps_rd_max),
4126         bps_wr_max => int($bps_wr_max),
4127         iops_max => int($iops_max),
4128         iops_rd_max => int($iops_rd_max),
4129         iops_wr_max => int($iops_wr_max),
4130         bps_max_length => int($bps_max_length),
4131         bps_rd_max_length => int($bps_rd_max_length),
4132         bps_wr_max_length => int($bps_wr_max_length),
4133         iops_max_length => int($iops_max_length),
4134         iops_rd_max_length => int($iops_rd_max_length),
4135         iops_wr_max_length => int($iops_wr_max_length),
4136     );
4137
4138 }
4139
4140 # old code, only used to shutdown old VM after update
4141 sub __read_avail {
4142     my ($fh, $timeout) = @_;
4143
4144     my $sel = new IO::Select;
4145     $sel->add($fh);
4146
4147     my $res = '';
4148     my $buf;
4149
4150     my @ready;
4151     while (scalar (@ready = $sel->can_read($timeout))) {
4152         my $count;
4153         if ($count = $fh->sysread($buf, 8192)) {
4154             if ($buf =~ /^(.*)\(qemu\) $/s) {
4155                 $res .= $1;
4156                 last;
4157             } else {
4158                 $res .= $buf;
4159             }
4160         } else {
4161             if (!defined($count)) {
4162                 die "$!\n";
4163             }
4164             last;
4165         }
4166     }
4167
4168     die "monitor read timeout\n" if !scalar(@ready);
4169
4170     return $res;
4171 }
4172
4173 sub qemu_block_resize {
4174     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4175
4176     my $running = check_running($vmid);
4177
4178     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4179
4180     return if !$running;
4181
4182     my $padding = (1024 - $size % 1024) % 1024;
4183     $size = $size + $padding;
4184
4185     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4186
4187 }
4188
4189 sub qemu_volume_snapshot {
4190     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4191
4192     my $running = check_running($vmid);
4193
4194     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4195         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4196     } else {
4197         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4198     }
4199 }
4200
4201 sub qemu_volume_snapshot_delete {
4202     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4203
4204     my $running = check_running($vmid);
4205
4206     if($running) {
4207
4208         $running = undef;
4209         my $conf = PVE::QemuConfig->load_config($vmid);
4210         foreach_drive($conf, sub {
4211             my ($ds, $drive) = @_;
4212             $running = 1 if $drive->{file} eq $volid;
4213         });
4214     }
4215
4216     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4217         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4218     } else {
4219         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4220     }
4221 }
4222
4223 sub set_migration_caps {
4224     my ($vmid) = @_;
4225
4226     my $cap_ref = [];
4227
4228     my $enabled_cap = {
4229         "auto-converge" => 1,
4230         "xbzrle" => 1,
4231         "x-rdma-pin-all" => 0,
4232         "zero-blocks" => 0,
4233         "compress" => 0
4234     };
4235
4236     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4237
4238     for my $supported_capability (@$supported_capabilities) {
4239         push @$cap_ref, {
4240             capability => $supported_capability->{capability},
4241             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4242         };
4243     }
4244
4245     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4246 }
4247
4248 my $fast_plug_option = {
4249     'lock' => 1,
4250     'name' => 1,
4251     'onboot' => 1,
4252     'shares' => 1,
4253     'startup' => 1,
4254     'description' => 1,
4255     'protection' => 1,
4256     'vmstatestorage' => 1,
4257     'hookscript' => 1,
4258     'tags' => 1,
4259 };
4260
4261 # hotplug changes in [PENDING]
4262 # $selection hash can be used to only apply specified options, for
4263 # example: { cores => 1 } (only apply changed 'cores')
4264 # $errors ref is used to return error messages
4265 sub vmconfig_hotplug_pending {
4266     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4267
4268     my $defaults = load_defaults();
4269     my $arch = get_vm_arch($conf);
4270     my $machine_type = get_vm_machine($conf, undef, $arch);
4271
4272     # commit values which do not have any impact on running VM first
4273     # Note: those option cannot raise errors, we we do not care about
4274     # $selection and always apply them.
4275
4276     my $add_error = sub {
4277         my ($opt, $msg) = @_;
4278         $errors->{$opt} = "hotplug problem - $msg";
4279     };
4280
4281     my $changes = 0;
4282     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4283         if ($fast_plug_option->{$opt}) {
4284             $conf->{$opt} = $conf->{pending}->{$opt};
4285             delete $conf->{pending}->{$opt};
4286             $changes = 1;
4287         }
4288     }
4289
4290     if ($changes) {
4291         PVE::QemuConfig->write_config($vmid, $conf);
4292     }
4293
4294     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4295
4296     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4297     foreach my $opt (sort keys %$pending_delete_hash) {
4298         next if $selection && !$selection->{$opt};
4299         my $force = $pending_delete_hash->{$opt}->{force};
4300         eval {
4301             if ($opt eq 'hotplug') {
4302                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4303             } elsif ($opt eq 'tablet') {
4304                 die "skip\n" if !$hotplug_features->{usb};
4305                 if ($defaults->{tablet}) {
4306                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4307                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4308                         if $arch eq 'aarch64';
4309                 } else {
4310                     vm_deviceunplug($vmid, $conf, 'tablet');
4311                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4312                 }
4313             } elsif ($opt =~ m/^usb\d+/) {
4314                 die "skip\n";
4315                 # since we cannot reliably hot unplug usb devices
4316                 # we are disabling it
4317                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4318                 vm_deviceunplug($vmid, $conf, $opt);
4319             } elsif ($opt eq 'vcpus') {
4320                 die "skip\n" if !$hotplug_features->{cpu};
4321                 qemu_cpu_hotplug($vmid, $conf, undef);
4322             } elsif ($opt eq 'balloon') {
4323                 # enable balloon device is not hotpluggable
4324                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4325                 # here we reset the ballooning value to memory
4326                 my $balloon = $conf->{memory} || $defaults->{memory};
4327                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4328             } elsif ($fast_plug_option->{$opt}) {
4329                 # do nothing
4330             } elsif ($opt =~ m/^net(\d+)$/) {
4331                 die "skip\n" if !$hotplug_features->{network};
4332                 vm_deviceunplug($vmid, $conf, $opt);
4333             } elsif (is_valid_drivename($opt)) {
4334                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4335                 vm_deviceunplug($vmid, $conf, $opt);
4336                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4337             } elsif ($opt =~ m/^memory$/) {
4338                 die "skip\n" if !$hotplug_features->{memory};
4339                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4340             } elsif ($opt eq 'cpuunits') {
4341                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4342             } elsif ($opt eq 'cpulimit') {
4343                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4344             } else {
4345                 die "skip\n";
4346             }
4347         };
4348         if (my $err = $@) {
4349             &$add_error($opt, $err) if $err ne "skip\n";
4350         } else {
4351             delete $conf->{$opt};
4352             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4353         }
4354     }
4355
4356     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4357     $apply_pending_cloudinit = sub {
4358         return if $apply_pending_cloudinit_done; # once is enough
4359         $apply_pending_cloudinit_done = 1; # once is enough
4360
4361         my ($key, $value) = @_;
4362
4363         my @cloudinit_opts = keys %$confdesc_cloudinit;
4364         foreach my $opt (keys %{$conf->{pending}}) {
4365             next if !grep { $_ eq $opt } @cloudinit_opts;
4366             $conf->{$opt} = delete $conf->{pending}->{$opt};
4367         }
4368
4369         my $new_conf = { %$conf };
4370         $new_conf->{$key} = $value;
4371         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4372     };
4373
4374     foreach my $opt (keys %{$conf->{pending}}) {
4375         next if $selection && !$selection->{$opt};
4376         my $value = $conf->{pending}->{$opt};
4377         eval {
4378             if ($opt eq 'hotplug') {
4379                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4380             } elsif ($opt eq 'tablet') {
4381                 die "skip\n" if !$hotplug_features->{usb};
4382                 if ($value == 1) {
4383                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4384                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4385                         if $arch eq 'aarch64';
4386                 } elsif ($value == 0) {
4387                     vm_deviceunplug($vmid, $conf, 'tablet');
4388                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4389                 }
4390             } elsif ($opt =~ m/^usb\d+$/) {
4391                 die "skip\n";
4392                 # since we cannot reliably hot unplug usb devices
4393                 # we are disabling it
4394                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4395                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4396                 die "skip\n" if !$d;
4397                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4398             } elsif ($opt eq 'vcpus') {
4399                 die "skip\n" if !$hotplug_features->{cpu};
4400                 qemu_cpu_hotplug($vmid, $conf, $value);
4401             } elsif ($opt eq 'balloon') {
4402                 # enable/disable balloning device is not hotpluggable
4403                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4404                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4405                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4406
4407                 # allow manual ballooning if shares is set to zero
4408                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4409                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4410                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4411                 }
4412             } elsif ($opt =~ m/^net(\d+)$/) {
4413                 # some changes can be done without hotplug
4414                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4415                                     $vmid, $opt, $value, $arch, $machine_type);
4416             } elsif (is_valid_drivename($opt)) {
4417                 die "skip\n" if $opt eq 'efidisk0';
4418                 # some changes can be done without hotplug
4419                 my $drive = parse_drive($opt, $value);
4420                 if (drive_is_cloudinit($drive)) {
4421                     &$apply_pending_cloudinit($opt, $value);
4422                 }
4423                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4424                                      $vmid, $opt, $value, $arch, $machine_type);
4425             } elsif ($opt =~ m/^memory$/) { #dimms
4426                 die "skip\n" if !$hotplug_features->{memory};
4427                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4428             } elsif ($opt eq 'cpuunits') {
4429                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4430             } elsif ($opt eq 'cpulimit') {
4431                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4432                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4433             } else {
4434                 die "skip\n";  # skip non-hot-pluggable options
4435             }
4436         };
4437         if (my $err = $@) {
4438             &$add_error($opt, $err) if $err ne "skip\n";
4439         } else {
4440             $conf->{$opt} = $value;
4441             delete $conf->{pending}->{$opt};
4442         }
4443     }
4444
4445     PVE::QemuConfig->write_config($vmid, $conf);
4446 }
4447
4448 sub try_deallocate_drive {
4449     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4450
4451     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4452         my $volid = $drive->{file};
4453         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4454             my $sid = PVE::Storage::parse_volume_id($volid);
4455             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4456
4457             # check if the disk is really unused
4458             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4459                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4460             PVE::Storage::vdisk_free($storecfg, $volid);
4461             return 1;
4462         } else {
4463             # If vm is not owner of this disk remove from config
4464             return 1;
4465         }
4466     }
4467
4468     return undef;
4469 }
4470
4471 sub vmconfig_delete_or_detach_drive {
4472     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4473
4474     my $drive = parse_drive($opt, $conf->{$opt});
4475
4476     my $rpcenv = PVE::RPCEnvironment::get();
4477     my $authuser = $rpcenv->get_user();
4478
4479     if ($force) {
4480         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4481         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4482     } else {
4483         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4484     }
4485 }
4486
4487
4488
4489 sub vmconfig_apply_pending {
4490     my ($vmid, $conf, $storecfg, $errors) = @_;
4491
4492     my $add_apply_error = sub {
4493         my ($opt, $msg) = @_;
4494         my $err_msg = "unable to apply pending change $opt : $msg";
4495         $errors->{$opt} = $err_msg;
4496         warn $err_msg;
4497     };
4498
4499     # cold plug
4500
4501     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4502     foreach my $opt (sort keys %$pending_delete_hash) {
4503         my $force = $pending_delete_hash->{$opt}->{force};
4504         eval {
4505             if ($opt =~ m/^unused/) {
4506                 die "internal error";
4507             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4508                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4509             }
4510         };
4511         if (my $err = $@) {
4512             $add_apply_error->($opt, $err);
4513         } else {
4514             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4515             delete $conf->{$opt};
4516         }
4517     }
4518
4519     PVE::QemuConfig->cleanup_pending($conf);
4520
4521     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4522         next if $opt eq 'delete'; # just to be sure
4523         eval {
4524             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4525                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4526             }
4527         };
4528         if (my $err = $@) {
4529             $add_apply_error->($opt, $err);
4530         } else {
4531             $conf->{$opt} = delete $conf->{pending}->{$opt};
4532         }
4533     }
4534
4535     # write all changes at once to avoid unnecessary i/o
4536     PVE::QemuConfig->write_config($vmid, $conf);
4537 }
4538
4539 sub vmconfig_update_net {
4540     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4541
4542     my $newnet = parse_net($value);
4543
4544     if ($conf->{$opt}) {
4545         my $oldnet = parse_net($conf->{$opt});
4546
4547         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4548             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4549             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4550             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4551
4552             # for non online change, we try to hot-unplug
4553             die "skip\n" if !$hotplug;
4554             vm_deviceunplug($vmid, $conf, $opt);
4555         } else {
4556
4557             die "internal error" if $opt !~ m/net(\d+)/;
4558             my $iface = "tap${vmid}i$1";
4559
4560             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4561                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4562                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4563                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4564                 PVE::Network::tap_unplug($iface);
4565                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4566             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4567                 # Rate can be applied on its own but any change above needs to
4568                 # include the rate in tap_plug since OVS resets everything.
4569                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4570             }
4571
4572             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4573                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4574             }
4575
4576             return 1;
4577         }
4578     }
4579
4580     if ($hotplug) {
4581         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4582     } else {
4583         die "skip\n";
4584     }
4585 }
4586
4587 sub vmconfig_update_disk {
4588     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4589
4590     my $drive = parse_drive($opt, $value);
4591
4592     if ($conf->{$opt}) {
4593
4594         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4595
4596             my $media = $drive->{media} || 'disk';
4597             my $oldmedia = $old_drive->{media} || 'disk';
4598             die "unable to change media type\n" if $media ne $oldmedia;
4599
4600             if (!drive_is_cdrom($old_drive)) {
4601
4602                 if ($drive->{file} ne $old_drive->{file}) {
4603
4604                     die "skip\n" if !$hotplug;
4605
4606                     # unplug and register as unused
4607                     vm_deviceunplug($vmid, $conf, $opt);
4608                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4609
4610                 } else {
4611                     # update existing disk
4612
4613                     # skip non hotpluggable value
4614                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4615                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4616                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4617                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4618                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4619                         die "skip\n";
4620                     }
4621
4622                     # apply throttle
4623                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4624                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4625                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4626                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4627                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4628                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4629                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4630                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4631                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4632                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4633                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4634                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4635                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4636                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4637                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4638                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4639                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4640                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4641
4642                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4643                                                    ($drive->{mbps} || 0)*1024*1024,
4644                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4645                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4646                                                    $drive->{iops} || 0,
4647                                                    $drive->{iops_rd} || 0,
4648                                                    $drive->{iops_wr} || 0,
4649                                                    ($drive->{mbps_max} || 0)*1024*1024,
4650                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4651                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4652                                                    $drive->{iops_max} || 0,
4653                                                    $drive->{iops_rd_max} || 0,
4654                                                    $drive->{iops_wr_max} || 0,
4655                                                    $drive->{bps_max_length} || 1,
4656                                                    $drive->{bps_rd_max_length} || 1,
4657                                                    $drive->{bps_wr_max_length} || 1,
4658                                                    $drive->{iops_max_length} || 1,
4659                                                    $drive->{iops_rd_max_length} || 1,
4660                                                    $drive->{iops_wr_max_length} || 1);
4661
4662                     }
4663
4664                     return 1;
4665                 }
4666
4667             } else { # cdrom
4668
4669                 if ($drive->{file} eq 'none') {
4670                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4671                     if (drive_is_cloudinit($old_drive)) {
4672                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4673                     }
4674                 } else {
4675                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4676
4677                     # force eject if locked
4678                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4679
4680                     if ($path) {
4681                         mon_cmd($vmid, "blockdev-change-medium",
4682                             id => "$opt", filename => "$path");
4683                     }
4684                 }
4685
4686                 return 1;
4687             }
4688         }
4689     }
4690
4691     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4692     # hotplug new disks
4693     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4694     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4695 }
4696
4697 sub vm_start {
4698     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4699         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
4700
4701     PVE::QemuConfig->lock_config($vmid, sub {
4702         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4703
4704         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4705
4706         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
4707
4708         PVE::QemuConfig->check_lock($conf)
4709             if !($skiplock || $is_suspended);
4710
4711         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4712
4713         # clean up leftover reboot request files
4714         eval { clear_reboot_request($vmid); };
4715         warn $@ if $@;
4716
4717         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4718             vmconfig_apply_pending($vmid, $conf, $storecfg);
4719             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4720         }
4721
4722         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4723
4724         my $defaults = load_defaults();
4725
4726         # set environment variable useful inside network script
4727         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4728
4729         my $local_volumes = {};
4730
4731         if ($targetstorage) {
4732             foreach_drive($conf, sub {
4733                 my ($ds, $drive) = @_;
4734
4735                 return if drive_is_cdrom($drive);
4736
4737                 my $volid = $drive->{file};
4738
4739                 return if !$volid;
4740
4741                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4742
4743                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4744                 return if $scfg->{shared};
4745                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4746             });
4747
4748             my $format = undef;
4749
4750             foreach my $opt (sort keys %$local_volumes) {
4751
4752                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4753                 my $drive = parse_drive($opt, $conf->{$opt});
4754
4755                 # If a remote storage is specified and the format of the original
4756                 # volume is not available there, fall back to the default format.
4757                 # Otherwise use the same format as the original.
4758                 if ($targetstorage && $targetstorage ne "1") {
4759                     $storeid = $targetstorage;
4760                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4761                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4762                     my $fileFormat = qemu_img_format($scfg, $volname);
4763                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4764                 } else {
4765                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4766                     $format = qemu_img_format($scfg, $volname);
4767                 }
4768
4769                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4770                 my $newdrive = $drive;
4771                 $newdrive->{format} = $format;
4772                 $newdrive->{file} = $newvolid;
4773                 my $drivestr = print_drive($newdrive);
4774                 $local_volumes->{$opt} = $drivestr;
4775                 #pass drive to conf for command line
4776                 $conf->{$opt} = $drivestr;
4777             }
4778         }
4779
4780         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4781
4782         if ($is_suspended) {
4783             # enforce machine type on suspended vm to ensure HW compatibility
4784             $forcemachine = $conf->{runningmachine};
4785             print "Resuming suspended VM\n";
4786         }
4787
4788         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4789
4790         my $migration_ip;
4791         my $get_migration_ip = sub {
4792             my ($cidr, $nodename) = @_;
4793
4794             return $migration_ip if defined($migration_ip);
4795
4796             if (!defined($cidr)) {
4797                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4798                 $cidr = $dc_conf->{migration}->{network};
4799             }
4800
4801             if (defined($cidr)) {
4802                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4803
4804                 die "could not get IP: no address configured on local " .
4805                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
4806
4807                 die "could not get IP: multiple addresses configured on local " .
4808                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
4809
4810                 $migration_ip = @$ips[0];
4811             }
4812
4813             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4814                 if !defined($migration_ip);
4815
4816             return $migration_ip;
4817         };
4818
4819         my $migrate_uri;
4820         if ($statefile) {
4821             if ($statefile eq 'tcp') {
4822                 my $localip = "localhost";
4823                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4824                 my $nodename = nodename();
4825
4826                 if (!defined($migration_type)) {
4827                     if (defined($datacenterconf->{migration}->{type})) {
4828                         $migration_type = $datacenterconf->{migration}->{type};
4829                     } else {
4830                         $migration_type = 'secure';
4831                     }
4832                 }
4833
4834                 if ($migration_type eq 'insecure') {
4835                     $localip = $get_migration_ip->($migration_network, $nodename);
4836                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4837                 }
4838
4839                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4840                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4841                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4842                 push @$cmd, '-incoming', $migrate_uri;
4843                 push @$cmd, '-S';
4844
4845             } elsif ($statefile eq 'unix') {
4846                 # should be default for secure migrations as a ssh TCP forward
4847                 # tunnel is not deterministic reliable ready and fails regurarly
4848                 # to set up in time, so use UNIX socket forwards
4849                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4850                 unlink $socket_addr;
4851
4852                 $migrate_uri = "unix:$socket_addr";
4853
4854                 push @$cmd, '-incoming', $migrate_uri;
4855                 push @$cmd, '-S';
4856
4857             } elsif (-e $statefile) {
4858                 push @$cmd, '-loadstate', $statefile;
4859             } else {
4860                 my $statepath = PVE::Storage::path($storecfg, $statefile);
4861                 push @$vollist, $statefile;
4862                 push @$cmd, '-loadstate', $statepath;
4863             }
4864         } elsif ($paused) {
4865             push @$cmd, '-S';
4866         }
4867
4868         # host pci devices
4869         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4870           my $d = parse_hostpci($conf->{"hostpci$i"});
4871           next if !$d;
4872           my $pcidevices = $d->{pciid};
4873           foreach my $pcidevice (@$pcidevices) {
4874                 my $pciid = $pcidevice->{id};
4875
4876                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4877                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4878                 die "no pci device info for device '$pciid'\n" if !$info;
4879
4880                 if ($d->{mdev}) {
4881                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4882                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4883                 } else {
4884                     die "can't unbind/bind pci group to vfio '$pciid'\n"
4885                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4886                     die "can't reset pci device '$pciid'\n"
4887                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4888                 }
4889           }
4890         }
4891
4892         PVE::Storage::activate_volumes($storecfg, $vollist);
4893
4894         eval {
4895             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4896                 outfunc => sub {}, errfunc => sub {});
4897         };
4898         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4899         # timeout should be more than enough here...
4900         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4901
4902         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4903                                                   : $defaults->{cpuunits};
4904
4905         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
4906         my %run_params = (
4907             timeout => $statefile ? undef : $start_timeout,
4908             umask => 0077,
4909             noerr => 1,
4910         );
4911
4912         # when migrating, prefix QEMU output so other side can pick up any
4913         # errors that might occur and show the user
4914         if ($migratedfrom) {
4915             $run_params{quiet} = 1;
4916             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4917         }
4918
4919         my %properties = (
4920             Slice => 'qemu.slice',
4921             KillMode => 'none',
4922             CPUShares => $cpuunits
4923         );
4924
4925         if (my $cpulimit = $conf->{cpulimit}) {
4926             $properties{CPUQuota} = int($cpulimit * 100);
4927         }
4928         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4929
4930         my $run_qemu = sub {
4931             PVE::Tools::run_fork sub {
4932                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4933
4934                 my $exitcode = run_command($cmd, %run_params);
4935                 die "QEMU exited with code $exitcode\n" if $exitcode;
4936             };
4937         };
4938
4939         if ($conf->{hugepages}) {
4940
4941             my $code = sub {
4942                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4943                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4944
4945                 PVE::QemuServer::Memory::hugepages_mount();
4946                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4947
4948                 eval { $run_qemu->() };
4949                 if (my $err = $@) {
4950                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4951                     die $err;
4952                 }
4953
4954                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4955             };
4956             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4957
4958         } else {
4959             eval { $run_qemu->() };
4960         }
4961
4962         if (my $err = $@) {
4963             # deactivate volumes if start fails
4964             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4965             die "start failed: $err";
4966         }
4967
4968         print "migration listens on $migrate_uri\n" if $migrate_uri;
4969
4970         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4971             eval { mon_cmd($vmid, "cont"); };
4972             warn $@ if $@;
4973         }
4974
4975         #start nbd server for storage migration
4976         if ($targetstorage) {
4977             my $nodename = nodename();
4978             my $localip = $get_migration_ip->($migration_network, $nodename);
4979             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4980             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
4981
4982             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
4983
4984             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4985
4986             foreach my $opt (sort keys %$local_volumes) {
4987                 my $drivestr = $local_volumes->{$opt};
4988                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4989                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
4990                 print "storage migration listens on $migrate_storage_uri volume:$drivestr\n";
4991             }
4992         }
4993
4994         if ($migratedfrom) {
4995             eval {
4996                 set_migration_caps($vmid);
4997             };
4998             warn $@ if $@;
4999
5000             if ($spice_port) {
5001                 print "spice listens on port $spice_port\n";
5002                 if ($spice_ticket) {
5003                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5004                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5005                 }
5006             }
5007
5008         } else {
5009             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5010                 if !$statefile && $conf->{balloon};
5011
5012             foreach my $opt (keys %$conf) {
5013                 next if $opt !~  m/^net\d+$/;
5014                 my $nicconf = parse_net($conf->{$opt});
5015                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5016             }
5017         }
5018
5019         mon_cmd($vmid, 'qom-set',
5020                     path => "machine/peripheral/balloon0",
5021                     property => "guest-stats-polling-interval",
5022                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5023
5024         if ($is_suspended) {
5025             print "Resumed VM, removing state\n";
5026             if (my $vmstate = $conf->{vmstate}) {
5027                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5028                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5029             }
5030             delete $conf->@{qw(lock vmstate runningmachine)};
5031             PVE::QemuConfig->write_config($vmid, $conf);
5032         }
5033
5034         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5035     });
5036 }
5037
5038 sub vm_commandline {
5039     my ($storecfg, $vmid, $snapname) = @_;
5040
5041     my $conf = PVE::QemuConfig->load_config($vmid);
5042     my $forcemachine;
5043
5044     if ($snapname) {
5045         my $snapshot = $conf->{snapshots}->{$snapname};
5046         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5047
5048         # check for a 'runningmachine' in snapshot
5049         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5050
5051         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5052
5053         $conf = $snapshot;
5054     }
5055
5056     my $defaults = load_defaults();
5057
5058     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5059
5060     return PVE::Tools::cmd2string($cmd);
5061 }
5062
5063 sub vm_reset {
5064     my ($vmid, $skiplock) = @_;
5065
5066     PVE::QemuConfig->lock_config($vmid, sub {
5067
5068         my $conf = PVE::QemuConfig->load_config($vmid);
5069
5070         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5071
5072         mon_cmd($vmid, "system_reset");
5073     });
5074 }
5075
5076 sub get_vm_volumes {
5077     my ($conf) = @_;
5078
5079     my $vollist = [];
5080     foreach_volid($conf, sub {
5081         my ($volid, $attr) = @_;
5082
5083         return if $volid =~ m|^/|;
5084
5085         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5086         return if !$sid;
5087
5088         push @$vollist, $volid;
5089     });
5090
5091     return $vollist;
5092 }
5093
5094 sub vm_stop_cleanup {
5095     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5096
5097     eval {
5098
5099         if (!$keepActive) {
5100             my $vollist = get_vm_volumes($conf);
5101             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5102         }
5103
5104         foreach my $ext (qw(mon qmp pid vnc qga)) {
5105             unlink "/var/run/qemu-server/${vmid}.$ext";
5106         }
5107
5108         if ($conf->{ivshmem}) {
5109             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5110             # just delete it for now, VMs which have this already open do not
5111             # are affected, but new VMs will get a separated one. If this
5112             # becomes an issue we either add some sort of ref-counting or just
5113             # add a "don't delete on stop" flag to the ivshmem format.
5114             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5115         }
5116
5117         foreach my $key (keys %$conf) {
5118             next if $key !~ m/^hostpci(\d+)$/;
5119             my $hostpciindex = $1;
5120             my $d = parse_hostpci($conf->{$key});
5121             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5122
5123             foreach my $pci (@{$d->{pciid}}) {
5124                 my $pciid = $pci->{id};
5125                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5126             }
5127         }
5128
5129         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5130     };
5131     warn $@ if $@; # avoid errors - just warn
5132 }
5133
5134 # call only in locked context
5135 sub _do_vm_stop {
5136     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5137
5138     my $pid = check_running($vmid, $nocheck);
5139     return if !$pid;
5140
5141     my $conf;
5142     if (!$nocheck) {
5143         $conf = PVE::QemuConfig->load_config($vmid);
5144         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5145         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5146             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5147             $timeout = $opts->{down} if $opts->{down};
5148         }
5149         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5150     }
5151
5152     eval {
5153         if ($shutdown) {
5154             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5155                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5156             } else {
5157                 mon_cmd($vmid, "system_powerdown");
5158             }
5159         } else {
5160             mon_cmd($vmid, "quit");
5161         }
5162     };
5163     my $err = $@;
5164
5165     if (!$err) {
5166         $timeout = 60 if !defined($timeout);
5167
5168         my $count = 0;
5169         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5170             $count++;
5171             sleep 1;
5172         }
5173
5174         if ($count >= $timeout) {
5175             if ($force) {
5176                 warn "VM still running - terminating now with SIGTERM\n";
5177                 kill 15, $pid;
5178             } else {
5179                 die "VM quit/powerdown failed - got timeout\n";
5180             }
5181         } else {
5182             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5183             return;
5184         }
5185     } else {
5186         if ($force) {
5187             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5188             kill 15, $pid;
5189         } else {
5190             die "VM quit/powerdown failed\n";
5191         }
5192     }
5193
5194     # wait again
5195     $timeout = 10;
5196
5197     my $count = 0;
5198     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5199         $count++;
5200         sleep 1;
5201     }
5202
5203     if ($count >= $timeout) {
5204         warn "VM still running - terminating now with SIGKILL\n";
5205         kill 9, $pid;
5206         sleep 1;
5207     }
5208
5209     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5210 }
5211
5212 # Note: use $nocheck to skip tests if VM configuration file exists.
5213 # We need that when migration VMs to other nodes (files already moved)
5214 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5215 sub vm_stop {
5216     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5217
5218     $force = 1 if !defined($force) && !$shutdown;
5219
5220     if ($migratedfrom){
5221         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5222         kill 15, $pid if $pid;
5223         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5224         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5225         return;
5226     }
5227
5228     PVE::QemuConfig->lock_config($vmid, sub {
5229         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5230    });
5231 }
5232
5233 sub vm_reboot {
5234     my ($vmid, $timeout) = @_;
5235
5236     PVE::QemuConfig->lock_config($vmid, sub {
5237         eval {
5238
5239             # only reboot if running, as qmeventd starts it again on a stop event
5240             return if !check_running($vmid);
5241
5242             create_reboot_request($vmid);
5243
5244             my $storecfg = PVE::Storage::config();
5245             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5246
5247         };
5248         if (my $err = $@) {
5249             # avoid that the next normal shutdown will be confused for a reboot
5250             clear_reboot_request($vmid);
5251             die $err;
5252         }
5253    });
5254 }
5255
5256 # note: if using the statestorage parameter, the caller has to check privileges
5257 sub vm_suspend {
5258     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5259
5260     my $conf;
5261     my $path;
5262     my $storecfg;
5263     my $vmstate;
5264
5265     PVE::QemuConfig->lock_config($vmid, sub {
5266
5267         $conf = PVE::QemuConfig->load_config($vmid);
5268
5269         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5270         PVE::QemuConfig->check_lock($conf)
5271             if !($skiplock || $is_backing_up);
5272
5273         die "cannot suspend to disk during backup\n"
5274             if $is_backing_up && $includestate;
5275
5276         if ($includestate) {
5277             $conf->{lock} = 'suspending';
5278             my $date = strftime("%Y-%m-%d", localtime(time()));
5279             $storecfg = PVE::Storage::config();
5280             if (!$statestorage) {
5281                 $statestorage = find_vmstate_storage($conf, $storecfg);
5282                 # check permissions for the storage
5283                 my $rpcenv = PVE::RPCEnvironment::get();
5284                 if ($rpcenv->{type} ne 'cli') {
5285                     my $authuser = $rpcenv->get_user();
5286                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5287                 }
5288             }
5289
5290
5291             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5292             $path = PVE::Storage::path($storecfg, $vmstate);
5293             PVE::QemuConfig->write_config($vmid, $conf);
5294         } else {
5295             mon_cmd($vmid, "stop");
5296         }
5297     });
5298
5299     if ($includestate) {
5300         # save vm state
5301         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5302
5303         eval {
5304             mon_cmd($vmid, "savevm-start", statefile => $path);
5305             for(;;) {
5306                 my $state = mon_cmd($vmid, "query-savevm");
5307                 if (!$state->{status}) {
5308                     die "savevm not active\n";
5309                 } elsif ($state->{status} eq 'active') {
5310                     sleep(1);
5311                     next;
5312                 } elsif ($state->{status} eq 'completed') {
5313                     print "State saved, quitting\n";
5314                     last;
5315                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5316                     die "query-savevm failed with error '$state->{error}'\n"
5317                 } else {
5318                     die "query-savevm returned status '$state->{status}'\n";
5319                 }
5320             }
5321         };
5322         my $err = $@;
5323
5324         PVE::QemuConfig->lock_config($vmid, sub {
5325             $conf = PVE::QemuConfig->load_config($vmid);
5326             if ($err) {
5327                 # cleanup, but leave suspending lock, to indicate something went wrong
5328                 eval {
5329                     mon_cmd($vmid, "savevm-end");
5330                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5331                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5332                     delete $conf->@{qw(vmstate runningmachine)};
5333                     PVE::QemuConfig->write_config($vmid, $conf);
5334                 };
5335                 warn $@ if $@;
5336                 die $err;
5337             }
5338
5339             die "lock changed unexpectedly\n"
5340                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5341
5342             mon_cmd($vmid, "quit");
5343             $conf->{lock} = 'suspended';
5344             PVE::QemuConfig->write_config($vmid, $conf);
5345         });
5346     }
5347 }
5348
5349 sub vm_resume {
5350     my ($vmid, $skiplock, $nocheck) = @_;
5351
5352     PVE::QemuConfig->lock_config($vmid, sub {
5353         my $res = mon_cmd($vmid, 'query-status');
5354         my $resume_cmd = 'cont';
5355
5356         if ($res->{status} && $res->{status} eq 'suspended') {
5357             $resume_cmd = 'system_wakeup';
5358         }
5359
5360         if (!$nocheck) {
5361
5362             my $conf = PVE::QemuConfig->load_config($vmid);
5363
5364             PVE::QemuConfig->check_lock($conf)
5365                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5366         }
5367
5368         mon_cmd($vmid, $resume_cmd);
5369     });
5370 }
5371
5372 sub vm_sendkey {
5373     my ($vmid, $skiplock, $key) = @_;
5374
5375     PVE::QemuConfig->lock_config($vmid, sub {
5376
5377         my $conf = PVE::QemuConfig->load_config($vmid);
5378
5379         # there is no qmp command, so we use the human monitor command
5380         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5381         die $res if $res ne '';
5382     });
5383 }
5384
5385 # vzdump restore implementaion
5386
5387 sub tar_archive_read_firstfile {
5388     my $archive = shift;
5389
5390     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5391
5392     # try to detect archive type first
5393     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5394         die "unable to open file '$archive'\n";
5395     my $firstfile = <$fh>;
5396     kill 15, $pid;
5397     close $fh;
5398
5399     die "ERROR: archive contaions no data\n" if !$firstfile;
5400     chomp $firstfile;
5401
5402     return $firstfile;
5403 }
5404
5405 sub tar_restore_cleanup {
5406     my ($storecfg, $statfile) = @_;
5407
5408     print STDERR "starting cleanup\n";
5409
5410     if (my $fd = IO::File->new($statfile, "r")) {
5411         while (defined(my $line = <$fd>)) {
5412             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5413                 my $volid = $2;
5414                 eval {
5415                     if ($volid =~ m|^/|) {
5416                         unlink $volid || die 'unlink failed\n';
5417                     } else {
5418                         PVE::Storage::vdisk_free($storecfg, $volid);
5419                     }
5420                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5421                 };
5422                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5423             } else {
5424                 print STDERR "unable to parse line in statfile - $line";
5425             }
5426         }
5427         $fd->close();
5428     }
5429 }
5430
5431 sub restore_archive {
5432     my ($archive, $vmid, $user, $opts) = @_;
5433
5434     my $format = $opts->{format};
5435     my $comp;
5436
5437     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5438         $format = 'tar' if !$format;
5439         $comp = 'gzip';
5440     } elsif ($archive =~ m/\.tar$/) {
5441         $format = 'tar' if !$format;
5442     } elsif ($archive =~ m/.tar.lzo$/) {
5443         $format = 'tar' if !$format;
5444         $comp = 'lzop';
5445     } elsif ($archive =~ m/\.vma$/) {
5446         $format = 'vma' if !$format;
5447     } elsif ($archive =~ m/\.vma\.gz$/) {
5448         $format = 'vma' if !$format;
5449         $comp = 'gzip';
5450     } elsif ($archive =~ m/\.vma\.lzo$/) {
5451         $format = 'vma' if !$format;
5452         $comp = 'lzop';
5453     } else {
5454         $format = 'vma' if !$format; # default
5455     }
5456
5457     # try to detect archive format
5458     if ($format eq 'tar') {
5459         return restore_tar_archive($archive, $vmid, $user, $opts);
5460     } else {
5461         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5462     }
5463 }
5464
5465 sub restore_update_config_line {
5466     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5467
5468     return if $line =~ m/^\#qmdump\#/;
5469     return if $line =~ m/^\#vzdump\#/;
5470     return if $line =~ m/^lock:/;
5471     return if $line =~ m/^unused\d+:/;
5472     return if $line =~ m/^parent:/;
5473
5474     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5475     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5476         # try to convert old 1.X settings
5477         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5478         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5479             my ($model, $macaddr) = split(/\=/, $devconfig);
5480             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5481             my $net = {
5482                 model => $model,
5483                 bridge => "vmbr$ind",
5484                 macaddr => $macaddr,
5485             };
5486             my $netstr = print_net($net);
5487
5488             print $outfd "net$cookie->{netcount}: $netstr\n";
5489             $cookie->{netcount}++;
5490         }
5491     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5492         my ($id, $netstr) = ($1, $2);
5493         my $net = parse_net($netstr);
5494         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5495         $netstr = print_net($net);
5496         print $outfd "$id: $netstr\n";
5497     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5498         my $virtdev = $1;
5499         my $value = $3;
5500         my $di = parse_drive($virtdev, $value);
5501         if (defined($di->{backup}) && !$di->{backup}) {
5502             print $outfd "#$line";
5503         } elsif ($map->{$virtdev}) {
5504             delete $di->{format}; # format can change on restore
5505             $di->{file} = $map->{$virtdev};
5506             $value = print_drive($di);
5507             print $outfd "$virtdev: $value\n";
5508         } else {
5509             print $outfd $line;
5510         }
5511     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5512         my $vmgenid = $1;
5513         if ($vmgenid ne '0') {
5514             # always generate a new vmgenid if there was a valid one setup
5515             $vmgenid = generate_uuid();
5516         }
5517         print $outfd "vmgenid: $vmgenid\n";
5518     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5519         my ($uuid, $uuid_str);
5520         UUID::generate($uuid);
5521         UUID::unparse($uuid, $uuid_str);
5522         my $smbios1 = parse_smbios1($2);
5523         $smbios1->{uuid} = $uuid_str;
5524         print $outfd $1.print_smbios1($smbios1)."\n";
5525     } else {
5526         print $outfd $line;
5527     }
5528 }
5529
5530 sub scan_volids {
5531     my ($cfg, $vmid) = @_;
5532
5533     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5534
5535     my $volid_hash = {};
5536     foreach my $storeid (keys %$info) {
5537         foreach my $item (@{$info->{$storeid}}) {
5538             next if !($item->{volid} && $item->{size});
5539             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5540             $volid_hash->{$item->{volid}} = $item;
5541         }
5542     }
5543
5544     return $volid_hash;
5545 }
5546
5547 sub update_disk_config {
5548     my ($vmid, $conf, $volid_hash) = @_;
5549
5550     my $changes;
5551     my $prefix = "VM $vmid:";
5552
5553     # used and unused disks
5554     my $referenced = {};
5555
5556     # Note: it is allowed to define multiple storages with same path (alias), so
5557     # we need to check both 'volid' and real 'path' (two different volid can point
5558     # to the same path).
5559
5560     my $referencedpath = {};
5561
5562     # update size info
5563     foreach my $opt (keys %$conf) {
5564         if (is_valid_drivename($opt)) {
5565             my $drive = parse_drive($opt, $conf->{$opt});
5566             my $volid = $drive->{file};
5567             next if !$volid;
5568
5569             # mark volid as "in-use" for next step
5570             $referenced->{$volid} = 1;
5571             if ($volid_hash->{$volid} &&
5572                 (my $path = $volid_hash->{$volid}->{path})) {
5573                 $referencedpath->{$path} = 1;
5574             }
5575
5576             next if drive_is_cdrom($drive);
5577             next if !$volid_hash->{$volid};
5578
5579             my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5580             if (defined($updated)) {
5581                 $changes = 1;
5582                 $conf->{$opt} = print_drive($updated);
5583                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5584             }
5585         }
5586     }
5587
5588     # remove 'unusedX' entry if volume is used
5589     foreach my $opt (keys %$conf) {
5590         next if $opt !~ m/^unused\d+$/;
5591         my $volid = $conf->{$opt};
5592         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5593         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5594             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5595             $changes = 1;
5596             delete $conf->{$opt};
5597         }
5598
5599         $referenced->{$volid} = 1;
5600         $referencedpath->{$path} = 1 if $path;
5601     }
5602
5603     foreach my $volid (sort keys %$volid_hash) {
5604         next if $volid =~ m/vm-$vmid-state-/;
5605         next if $referenced->{$volid};
5606         my $path = $volid_hash->{$volid}->{path};
5607         next if !$path; # just to be sure
5608         next if $referencedpath->{$path};
5609         $changes = 1;
5610         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5611         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5612         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5613     }
5614
5615     return $changes;
5616 }
5617
5618 sub rescan {
5619     my ($vmid, $nolock, $dryrun) = @_;
5620
5621     my $cfg = PVE::Storage::config();
5622
5623     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5624     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5625     foreach my $stor (keys %{$cfg->{ids}}) {
5626         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5627     }
5628
5629     print "rescan volumes...\n";
5630     my $volid_hash = scan_volids($cfg, $vmid);
5631
5632     my $updatefn =  sub {
5633         my ($vmid) = @_;
5634
5635         my $conf = PVE::QemuConfig->load_config($vmid);
5636
5637         PVE::QemuConfig->check_lock($conf);
5638
5639         my $vm_volids = {};
5640         foreach my $volid (keys %$volid_hash) {
5641             my $info = $volid_hash->{$volid};
5642             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5643         }
5644
5645         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5646
5647         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5648     };
5649
5650     if (defined($vmid)) {
5651         if ($nolock) {
5652             &$updatefn($vmid);
5653         } else {
5654             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5655         }
5656     } else {
5657         my $vmlist = config_list();
5658         foreach my $vmid (keys %$vmlist) {
5659             if ($nolock) {
5660                 &$updatefn($vmid);
5661             } else {
5662                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5663             }
5664         }
5665     }
5666 }
5667
5668 sub restore_vma_archive {
5669     my ($archive, $vmid, $user, $opts, $comp) = @_;
5670
5671     my $readfrom = $archive;
5672
5673     my $cfg = PVE::Storage::config();
5674     my $commands = [];
5675     my $bwlimit = $opts->{bwlimit};
5676
5677     my $dbg_cmdstring = '';
5678     my $add_pipe = sub {
5679         my ($cmd) = @_;
5680         push @$commands, $cmd;
5681         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5682         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5683         $readfrom = '-';
5684     };
5685
5686     my $input = undef;
5687     if ($archive eq '-') {
5688         $input = '<&STDIN';
5689     } else {
5690         # If we use a backup from a PVE defined storage we also consider that
5691         # storage's rate limit:
5692         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5693         if (defined($volid)) {
5694             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5695             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5696             if ($readlimit) {
5697                 print STDERR "applying read rate limit: $readlimit\n";
5698                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5699                 $add_pipe->($cstream);
5700             }
5701         }
5702     }
5703
5704     if ($comp) {
5705         my $cmd;
5706         if ($comp eq 'gzip') {
5707             $cmd = ['zcat', $readfrom];
5708         } elsif ($comp eq 'lzop') {
5709             $cmd = ['lzop', '-d', '-c', $readfrom];
5710         } else {
5711             die "unknown compression method '$comp'\n";
5712         }
5713         $add_pipe->($cmd);
5714     }
5715
5716     my $tmpdir = "/var/tmp/vzdumptmp$$";
5717     rmtree $tmpdir;
5718
5719     # disable interrupts (always do cleanups)
5720     local $SIG{INT} =
5721         local $SIG{TERM} =
5722         local $SIG{QUIT} =
5723         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5724
5725     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5726     POSIX::mkfifo($mapfifo, 0600);
5727     my $fifofh;
5728
5729     my $openfifo = sub {
5730         open($fifofh, '>', $mapfifo) || die $!;
5731     };
5732
5733     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5734
5735     my $oldtimeout;
5736     my $timeout = 5;
5737
5738     my $devinfo = {};
5739
5740     my $rpcenv = PVE::RPCEnvironment::get();
5741
5742     my $conffile = PVE::QemuConfig->config_file($vmid);
5743     my $tmpfn = "$conffile.$$.tmp";
5744
5745     # Note: $oldconf is undef if VM does not exist
5746     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5747     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5748
5749     my %storage_limits;
5750
5751     my $print_devmap = sub {
5752         my $virtdev_hash = {};
5753
5754         my $cfgfn = "$tmpdir/qemu-server.conf";
5755
5756         # we can read the config - that is already extracted
5757         my $fh = IO::File->new($cfgfn, "r") ||
5758             "unable to read qemu-server.conf - $!\n";
5759
5760         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5761         if (-f $fwcfgfn) {
5762             my $pve_firewall_dir = '/etc/pve/firewall';
5763             mkdir $pve_firewall_dir; # make sure the dir exists
5764             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5765         }
5766
5767         while (defined(my $line = <$fh>)) {
5768             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5769                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5770                 die "archive does not contain data for drive '$virtdev'\n"
5771                     if !$devinfo->{$devname};
5772                 if (defined($opts->{storage})) {
5773                     $storeid = $opts->{storage} || 'local';
5774                 } elsif (!$storeid) {
5775                     $storeid = 'local';
5776                 }
5777                 $format = 'raw' if !$format;
5778                 $devinfo->{$devname}->{devname} = $devname;
5779                 $devinfo->{$devname}->{virtdev} = $virtdev;
5780                 $devinfo->{$devname}->{format} = $format;
5781                 $devinfo->{$devname}->{storeid} = $storeid;
5782
5783                 # check permission on storage
5784                 my $pool = $opts->{pool}; # todo: do we need that?
5785                 if ($user ne 'root@pam') {
5786                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5787                 }
5788
5789                 $storage_limits{$storeid} = $bwlimit;
5790
5791                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5792             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5793                 my $virtdev = $1;
5794                 my $drive = parse_drive($virtdev, $2);
5795                 if (drive_is_cloudinit($drive)) {
5796                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5797                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5798                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5799
5800                     my $d = {
5801                         format => $format,
5802                         storeid => $opts->{storage} // $storeid,
5803                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5804                         file => $drive->{file}, # to make drive_is_cloudinit check possible
5805                         name => "vm-$vmid-cloudinit",
5806                         is_cloudinit => 1,
5807                     };
5808                     $virtdev_hash->{$virtdev} = $d;
5809                 }
5810             }
5811         }
5812
5813         foreach my $key (keys %storage_limits) {
5814             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5815             next if !$limit;
5816             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5817             $storage_limits{$key} = $limit * 1024;
5818         }
5819
5820         foreach my $devname (keys %$devinfo) {
5821             die "found no device mapping information for device '$devname'\n"
5822                 if !$devinfo->{$devname}->{virtdev};
5823         }
5824
5825         # create empty/temp config
5826         if ($oldconf) {
5827             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5828             foreach_drive($oldconf, sub {
5829                 my ($ds, $drive) = @_;
5830
5831                 return if drive_is_cdrom($drive, 1);
5832
5833                 my $volid = $drive->{file};
5834                 return if !$volid || $volid =~ m|^/|;
5835
5836                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5837                 return if !$path || !$owner || ($owner != $vmid);
5838
5839                 # Note: only delete disk we want to restore
5840                 # other volumes will become unused
5841                 if ($virtdev_hash->{$ds}) {
5842                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5843                     if (my $err = $@) {
5844                         warn $err;
5845                     }
5846                 }
5847             });
5848
5849             # delete vmstate files, after the restore we have no snapshots anymore
5850             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5851                 my $snap = $oldconf->{snapshots}->{$snapname};
5852                 if ($snap->{vmstate}) {
5853                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5854                     if (my $err = $@) {
5855                         warn $err;
5856                     }
5857                 }
5858             }
5859         }
5860
5861         my $map = {};
5862         foreach my $virtdev (sort keys %$virtdev_hash) {
5863             my $d = $virtdev_hash->{$virtdev};
5864             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5865             my $storeid = $d->{storeid};
5866             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5867
5868             my $map_opts = '';
5869             if (my $limit = $storage_limits{$storeid}) {
5870                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5871             }
5872
5873             # test if requested format is supported
5874             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5875             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5876             $d->{format} = $defFormat if !$supported;
5877
5878             my $name;
5879             if ($d->{is_cloudinit}) {
5880                 $name = $d->{name};
5881                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
5882             }
5883
5884             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5885             print STDERR "new volume ID is '$volid'\n";
5886             $d->{volid} = $volid;
5887
5888             PVE::Storage::activate_volumes($cfg, [$volid]);
5889
5890             my $write_zeros = 1;
5891             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5892                 $write_zeros = 0;
5893             }
5894
5895             if (!$d->{is_cloudinit}) {
5896                 my $path = PVE::Storage::path($cfg, $volid);
5897
5898                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5899
5900                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5901             }
5902             $map->{$virtdev} = $volid;
5903         }
5904
5905         $fh->seek(0, 0) || die "seek failed - $!\n";
5906
5907         my $outfd = new IO::File ($tmpfn, "w") ||
5908             die "unable to write config for VM $vmid\n";
5909
5910         my $cookie = { netcount => 0 };
5911         while (defined(my $line = <$fh>)) {
5912             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5913         }
5914
5915         $fh->close();
5916         $outfd->close();
5917     };
5918
5919     eval {
5920         # enable interrupts
5921         local $SIG{INT} =
5922             local $SIG{TERM} =
5923             local $SIG{QUIT} =
5924             local $SIG{HUP} =
5925             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5926         local $SIG{ALRM} = sub { die "got timeout\n"; };
5927
5928         $oldtimeout = alarm($timeout);
5929
5930         my $parser = sub {
5931             my $line = shift;
5932
5933             print "$line\n";
5934
5935             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5936                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5937                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5938             } elsif ($line =~ m/^CTIME: /) {
5939                 # we correctly received the vma config, so we can disable
5940                 # the timeout now for disk allocation (set to 10 minutes, so
5941                 # that we always timeout if something goes wrong)
5942                 alarm(600);
5943                 &$print_devmap();
5944                 print $fifofh "done\n";
5945                 my $tmp = $oldtimeout || 0;
5946                 $oldtimeout = undef;
5947                 alarm($tmp);
5948                 close($fifofh);
5949             }
5950         };
5951
5952         print "restore vma archive: $dbg_cmdstring\n";
5953         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5954     };
5955     my $err = $@;
5956
5957     alarm($oldtimeout) if $oldtimeout;
5958
5959     my $vollist = [];
5960     foreach my $devname (keys %$devinfo) {
5961         my $volid = $devinfo->{$devname}->{volid};
5962         push @$vollist, $volid if $volid;
5963     }
5964
5965     PVE::Storage::deactivate_volumes($cfg, $vollist);
5966
5967     unlink $mapfifo;
5968
5969     if ($err) {
5970         rmtree $tmpdir;
5971         unlink $tmpfn;
5972
5973         foreach my $devname (keys %$devinfo) {
5974             my $volid = $devinfo->{$devname}->{volid};
5975             next if !$volid;
5976             eval {
5977                 if ($volid =~ m|^/|) {
5978                     unlink $volid || die 'unlink failed\n';
5979                 } else {
5980                     PVE::Storage::vdisk_free($cfg, $volid);
5981                 }
5982                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5983             };
5984             print STDERR "unable to cleanup '$volid' - $@" if $@;
5985         }
5986         die $err;
5987     }
5988
5989     rmtree $tmpdir;
5990
5991     rename($tmpfn, $conffile) ||
5992         die "unable to commit configuration file '$conffile'\n";
5993
5994     PVE::Cluster::cfs_update(); # make sure we read new file
5995
5996     eval { rescan($vmid, 1); };
5997     warn $@ if $@;
5998 }
5999
6000 sub restore_tar_archive {
6001     my ($archive, $vmid, $user, $opts) = @_;
6002
6003     if ($archive ne '-') {
6004         my $firstfile = tar_archive_read_firstfile($archive);
6005         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6006             if $firstfile ne 'qemu-server.conf';
6007     }
6008
6009     my $storecfg = PVE::Storage::config();
6010
6011     # avoid zombie disks when restoring over an existing VM -> cleanup first
6012     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6013     # skiplock=1 because qmrestore has set the 'create' lock itself already
6014     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6015     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6016
6017     my $tocmd = "/usr/lib/qemu-server/qmextract";
6018
6019     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6020     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6021     $tocmd .= ' --prealloc' if $opts->{prealloc};
6022     $tocmd .= ' --info' if $opts->{info};
6023
6024     # tar option "xf" does not autodetect compression when read from STDIN,
6025     # so we pipe to zcat
6026     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6027         PVE::Tools::shellquote("--to-command=$tocmd");
6028
6029     my $tmpdir = "/var/tmp/vzdumptmp$$";
6030     mkpath $tmpdir;
6031
6032     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6033     local $ENV{VZDUMP_VMID} = $vmid;
6034     local $ENV{VZDUMP_USER} = $user;
6035
6036     my $conffile = PVE::QemuConfig->config_file($vmid);
6037     my $tmpfn = "$conffile.$$.tmp";
6038
6039     # disable interrupts (always do cleanups)
6040     local $SIG{INT} =
6041         local $SIG{TERM} =
6042         local $SIG{QUIT} =
6043         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6044
6045     eval {
6046         # enable interrupts
6047         local $SIG{INT} =
6048             local $SIG{TERM} =
6049             local $SIG{QUIT} =
6050             local $SIG{HUP} =
6051             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6052
6053         if ($archive eq '-') {
6054             print "extracting archive from STDIN\n";
6055             run_command($cmd, input => "<&STDIN");
6056         } else {
6057             print "extracting archive '$archive'\n";
6058             run_command($cmd);
6059         }
6060
6061         return if $opts->{info};
6062
6063         # read new mapping
6064         my $map = {};
6065         my $statfile = "$tmpdir/qmrestore.stat";
6066         if (my $fd = IO::File->new($statfile, "r")) {
6067             while (defined (my $line = <$fd>)) {
6068                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6069                     $map->{$1} = $2 if $1;
6070                 } else {
6071                     print STDERR "unable to parse line in statfile - $line\n";
6072                 }
6073             }
6074             $fd->close();
6075         }
6076
6077         my $confsrc = "$tmpdir/qemu-server.conf";
6078
6079         my $srcfd = new IO::File($confsrc, "r") ||
6080             die "unable to open file '$confsrc'\n";
6081
6082         my $outfd = new IO::File ($tmpfn, "w") ||
6083             die "unable to write config for VM $vmid\n";
6084
6085         my $cookie = { netcount => 0 };
6086         while (defined (my $line = <$srcfd>)) {
6087             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6088         }
6089
6090         $srcfd->close();
6091         $outfd->close();
6092     };
6093     if (my $err = $@) {
6094         unlink $tmpfn;
6095         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6096         die $err;
6097     }
6098
6099     rmtree $tmpdir;
6100
6101     rename $tmpfn, $conffile ||
6102         die "unable to commit configuration file '$conffile'\n";
6103
6104     PVE::Cluster::cfs_update(); # make sure we read new file
6105
6106     eval { rescan($vmid, 1); };
6107     warn $@ if $@;
6108 };
6109
6110 sub foreach_storage_used_by_vm {
6111     my ($conf, $func) = @_;
6112
6113     my $sidhash = {};
6114
6115     foreach_drive($conf, sub {
6116         my ($ds, $drive) = @_;
6117         return if drive_is_cdrom($drive);
6118
6119         my $volid = $drive->{file};
6120
6121         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6122         $sidhash->{$sid} = $sid if $sid;
6123     });
6124
6125     foreach my $sid (sort keys %$sidhash) {
6126         &$func($sid);
6127     }
6128 }
6129
6130 my $qemu_snap_storage = {
6131     rbd => 1,
6132 };
6133 sub do_snapshots_with_qemu {
6134     my ($storecfg, $volid) = @_;
6135
6136     my $storage_name = PVE::Storage::parse_volume_id($volid);
6137     my $scfg = $storecfg->{ids}->{$storage_name};
6138
6139     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6140         return 1;
6141     }
6142
6143     if ($volid =~ m/\.(qcow2|qed)$/){
6144         return 1;
6145     }
6146
6147     return undef;
6148 }
6149
6150 sub qga_check_running {
6151     my ($vmid, $nowarn) = @_;
6152
6153     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6154     if ($@) {
6155         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6156         return 0;
6157     }
6158     return 1;
6159 }
6160
6161 sub template_create {
6162     my ($vmid, $conf, $disk) = @_;
6163
6164     my $storecfg = PVE::Storage::config();
6165
6166     foreach_drive($conf, sub {
6167         my ($ds, $drive) = @_;
6168
6169         return if drive_is_cdrom($drive);
6170         return if $disk && $ds ne $disk;
6171
6172         my $volid = $drive->{file};
6173         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6174
6175         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6176         $drive->{file} = $voliddst;
6177         $conf->{$ds} = print_drive($drive);
6178         PVE::QemuConfig->write_config($vmid, $conf);
6179     });
6180 }
6181
6182 sub convert_iscsi_path {
6183     my ($path) = @_;
6184
6185     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6186         my $portal = $1;
6187         my $target = $2;
6188         my $lun = $3;
6189
6190         my $initiator_name = get_initiator_name();
6191
6192         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6193                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6194     }
6195
6196     die "cannot convert iscsi path '$path', unkown format\n";
6197 }
6198
6199 sub qemu_img_convert {
6200     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6201
6202     my $storecfg = PVE::Storage::config();
6203     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6204     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6205
6206     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6207
6208     my $cachemode;
6209     my $src_path;
6210     my $src_is_iscsi = 0;
6211     my $src_format;
6212
6213     if ($src_storeid) {
6214         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6215         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6216         $src_format = qemu_img_format($src_scfg, $src_volname);
6217         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6218         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6219         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6220     } elsif (-f $src_volid) {
6221         $src_path = $src_volid;
6222         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6223             $src_format = $1;
6224         }
6225     }
6226
6227     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6228
6229     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6230     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6231     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6232     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6233
6234     my $cmd = [];
6235     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6236     push @$cmd, '-l', "snapshot.name=$snapname"
6237         if $snapname && $src_format && $src_format eq "qcow2";
6238     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6239     push @$cmd, '-T', $cachemode if defined($cachemode);
6240
6241     if ($src_is_iscsi) {
6242         push @$cmd, '--image-opts';
6243         $src_path = convert_iscsi_path($src_path);
6244     } elsif ($src_format) {
6245         push @$cmd, '-f', $src_format;
6246     }
6247
6248     if ($dst_is_iscsi) {
6249         push @$cmd, '--target-image-opts';
6250         $dst_path = convert_iscsi_path($dst_path);
6251     } else {
6252         push @$cmd, '-O', $dst_format;
6253     }
6254
6255     push @$cmd, $src_path;
6256
6257     if (!$dst_is_iscsi && $is_zero_initialized) {
6258         push @$cmd, "zeroinit:$dst_path";
6259     } else {
6260         push @$cmd, $dst_path;
6261     }
6262
6263     my $parser = sub {
6264         my $line = shift;
6265         if($line =~ m/\((\S+)\/100\%\)/){
6266             my $percent = $1;
6267             my $transferred = int($size * $percent / 100);
6268             my $remaining = $size - $transferred;
6269
6270             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6271         }
6272
6273     };
6274
6275     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6276     my $err = $@;
6277     die "copy failed: $err" if $err;
6278 }
6279
6280 sub qemu_img_format {
6281     my ($scfg, $volname) = @_;
6282
6283     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6284         return $1;
6285     } else {
6286         return "raw";
6287     }
6288 }
6289
6290 sub qemu_drive_mirror {
6291     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6292
6293     $jobs = {} if !$jobs;
6294
6295     my $qemu_target;
6296     my $format;
6297     $jobs->{"drive-$drive"} = {};
6298
6299     if ($dst_volid =~ /^nbd:/) {
6300         $qemu_target = $dst_volid;
6301         $format = "nbd";
6302     } else {
6303         my $storecfg = PVE::Storage::config();
6304         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6305
6306         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6307
6308         $format = qemu_img_format($dst_scfg, $dst_volname);
6309
6310         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6311
6312         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6313     }
6314
6315     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6316     $opts->{format} = $format if $format;
6317
6318     if (defined($bwlimit)) {
6319         $opts->{speed} = $bwlimit * 1024;
6320         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6321     } else {
6322         print "drive mirror is starting for drive-$drive\n";
6323     }
6324
6325     # if a job already runs for this device we get an error, catch it for cleanup
6326     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6327     if (my $err = $@) {
6328         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6329         warn "$@\n" if $@;
6330         die "mirroring error: $err\n";
6331     }
6332
6333     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6334 }
6335
6336 sub qemu_drive_mirror_monitor {
6337     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6338
6339     eval {
6340         my $err_complete = 0;
6341
6342         while (1) {
6343             die "storage migration timed out\n" if $err_complete > 300;
6344
6345             my $stats = mon_cmd($vmid, "query-block-jobs");
6346
6347             my $running_mirror_jobs = {};
6348             foreach my $stat (@$stats) {
6349                 next if $stat->{type} ne 'mirror';
6350                 $running_mirror_jobs->{$stat->{device}} = $stat;
6351             }
6352
6353             my $readycounter = 0;
6354
6355             foreach my $job (keys %$jobs) {
6356
6357                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6358                     print "$job : finished\n";
6359                     delete $jobs->{$job};
6360                     next;
6361                 }
6362
6363                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6364
6365                 my $busy = $running_mirror_jobs->{$job}->{busy};
6366                 my $ready = $running_mirror_jobs->{$job}->{ready};
6367                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6368                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6369                     my $remaining = $total - $transferred;
6370                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6371
6372                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6373                 }
6374
6375                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6376             }
6377
6378             last if scalar(keys %$jobs) == 0;
6379
6380             if ($readycounter == scalar(keys %$jobs)) {
6381                 print "all mirroring jobs are ready \n";
6382                 last if $skipcomplete; #do the complete later
6383
6384                 if ($vmiddst && $vmiddst != $vmid) {
6385                     my $agent_running = $qga && qga_check_running($vmid);
6386                     if ($agent_running) {
6387                         print "freeze filesystem\n";
6388                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6389                     } else {
6390                         print "suspend vm\n";
6391                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6392                     }
6393
6394                     # if we clone a disk for a new target vm, we don't switch the disk
6395                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6396
6397                     if ($agent_running) {
6398                         print "unfreeze filesystem\n";
6399                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6400                     } else {
6401                         print "resume vm\n";
6402                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6403                     }
6404
6405                     last;
6406                 } else {
6407
6408                     foreach my $job (keys %$jobs) {
6409                         # try to switch the disk if source and destination are on the same guest
6410                         print "$job: Completing block job...\n";
6411
6412                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6413                         if ($@ =~ m/cannot be completed/) {
6414                             print "$job: Block job cannot be completed, try again.\n";
6415                             $err_complete++;
6416                         }else {
6417                             print "$job: Completed successfully.\n";
6418                             $jobs->{$job}->{complete} = 1;
6419                         }
6420                     }
6421                 }
6422             }
6423             sleep 1;
6424         }
6425     };
6426     my $err = $@;
6427
6428     if ($err) {
6429         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6430         die "mirroring error: $err";
6431     }
6432
6433 }
6434
6435 sub qemu_blockjobs_cancel {
6436     my ($vmid, $jobs) = @_;
6437
6438     foreach my $job (keys %$jobs) {
6439         print "$job: Cancelling block job\n";
6440         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6441         $jobs->{$job}->{cancel} = 1;
6442     }
6443
6444     while (1) {
6445         my $stats = mon_cmd($vmid, "query-block-jobs");
6446
6447         my $running_jobs = {};
6448         foreach my $stat (@$stats) {
6449             $running_jobs->{$stat->{device}} = $stat;
6450         }
6451
6452         foreach my $job (keys %$jobs) {
6453
6454             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6455                 print "$job: Done.\n";
6456                 delete $jobs->{$job};
6457             }
6458         }
6459
6460         last if scalar(keys %$jobs) == 0;
6461
6462         sleep 1;
6463     }
6464 }
6465
6466 sub clone_disk {
6467     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6468         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6469
6470     my $newvolid;
6471
6472     if (!$full) {
6473         print "create linked clone of drive $drivename ($drive->{file})\n";
6474         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6475         push @$newvollist, $newvolid;
6476     } else {
6477
6478         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6479         $storeid = $storage if $storage;
6480
6481         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6482         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6483
6484         print "create full clone of drive $drivename ($drive->{file})\n";
6485         my $name = undef;
6486         if (drive_is_cloudinit($drive)) {
6487             $name = "vm-$newvmid-cloudinit";
6488             $name .= ".$dst_format" if $dst_format ne 'raw';
6489             $snapname = undef;
6490             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6491         }
6492         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6493         push @$newvollist, $newvolid;
6494
6495         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6496
6497         if (drive_is_cloudinit($drive)) {
6498             goto no_data_clone;
6499         }
6500
6501         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6502         if (!$running || $snapname) {
6503             # TODO: handle bwlimits
6504             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6505         } else {
6506
6507             my $kvmver = get_running_qemu_version ($vmid);
6508             if (!min_version($kvmver, 2, 7)) {
6509                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6510                     if $drive->{iothread};
6511             }
6512
6513             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6514         }
6515     }
6516
6517 no_data_clone:
6518     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6519
6520     my $disk = $drive;
6521     $disk->{format} = undef;
6522     $disk->{file} = $newvolid;
6523     $disk->{size} = $size;
6524
6525     return $disk;
6526 }
6527
6528 sub get_running_qemu_version {
6529     my ($vmid) = @_;
6530     my $res = mon_cmd($vmid, "query-version");
6531     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6532 }
6533
6534 sub qemu_use_old_bios_files {
6535     my ($machine_type) = @_;
6536
6537     return if !$machine_type;
6538
6539     my $use_old_bios_files = undef;
6540
6541     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6542         $machine_type = $1;
6543         $use_old_bios_files = 1;
6544     } else {
6545         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6546         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6547         # load new efi bios files on migration. So this hack is required to allow
6548         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6549         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6550         $use_old_bios_files = !min_version($version, 2, 4);
6551     }
6552
6553     return ($use_old_bios_files, $machine_type);
6554 }
6555
6556 sub create_efidisk($$$$$) {
6557     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6558
6559     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6560     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6561
6562     my $vars_size_b = -s $ovmf_vars;
6563     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6564     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6565     PVE::Storage::activate_volumes($storecfg, [$volid]);
6566
6567     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6568     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6569
6570     return ($volid, $size/1024);
6571 }
6572
6573 sub vm_iothreads_list {
6574     my ($vmid) = @_;
6575
6576     my $res = mon_cmd($vmid, 'query-iothreads');
6577
6578     my $iothreads = {};
6579     foreach my $iothread (@$res) {
6580         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6581     }
6582
6583     return $iothreads;
6584 }
6585
6586 sub scsihw_infos {
6587     my ($conf, $drive) = @_;
6588
6589     my $maxdev = 0;
6590
6591     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6592         $maxdev = 7;
6593     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6594         $maxdev = 1;
6595     } else {
6596         $maxdev = 256;
6597     }
6598
6599     my $controller = int($drive->{index} / $maxdev);
6600     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6601
6602     return ($maxdev, $controller, $controller_prefix);
6603 }
6604
6605 sub windows_version {
6606     my ($ostype) = @_;
6607
6608     return 0 if !$ostype;
6609
6610     my $winversion = 0;
6611
6612     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6613         $winversion = 5;
6614     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6615         $winversion = 6;
6616     } elsif ($ostype =~ m/^win(\d+)$/) {
6617         $winversion = $1;
6618     }
6619
6620     return $winversion;
6621 }
6622
6623 sub resolve_dst_disk_format {
6624         my ($storecfg, $storeid, $src_volname, $format) = @_;
6625         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6626
6627         if (!$format) {
6628             # if no target format is specified, use the source disk format as hint
6629             if ($src_volname) {
6630                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6631                 $format = qemu_img_format($scfg, $src_volname);
6632             } else {
6633                 return $defFormat;
6634             }
6635         }
6636
6637         # test if requested format is supported - else use default
6638         my $supported = grep { $_ eq $format } @$validFormats;
6639         $format = $defFormat if !$supported;
6640         return $format;
6641 }
6642
6643 # NOTE: if this logic changes, please update docs & possibly gui logic
6644 sub find_vmstate_storage {
6645     my ($conf, $storecfg) = @_;
6646
6647     # first, return storage from conf if set
6648     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
6649
6650     my ($target, $shared, $local);
6651
6652     foreach_storage_used_by_vm($conf, sub {
6653         my ($sid) = @_;
6654         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
6655         my $dst = $scfg->{shared} ? \$shared : \$local;
6656         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
6657     });
6658
6659     # second, use shared storage where VM has at least one disk
6660     # third, use local storage where VM has at least one disk
6661     # fall back to local storage
6662     $target = $shared // $local // 'local';
6663
6664     return $target;
6665 }
6666
6667 sub generate_uuid {
6668     my ($uuid, $uuid_str);
6669     UUID::generate($uuid);
6670     UUID::unparse($uuid, $uuid_str);
6671     return $uuid_str;
6672 }
6673
6674 sub generate_smbios1_uuid {
6675     return "uuid=".generate_uuid();
6676 }
6677
6678 sub nbd_stop {
6679     my ($vmid) = @_;
6680
6681     mon_cmd($vmid, 'nbd-server-stop');
6682 }
6683
6684 sub create_reboot_request {
6685     my ($vmid) = @_;
6686     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
6687         or die "failed to create reboot trigger file: $!\n";
6688     close($fh);
6689 }
6690
6691 sub clear_reboot_request {
6692     my ($vmid) = @_;
6693     my $path = "/run/qemu-server/$vmid.reboot";
6694     my $res = 0;
6695
6696     $res = unlink($path);
6697     die "could not remove reboot request for $vmid: $!"
6698         if !$res && $! != POSIX::ENOENT;
6699
6700     return $res;
6701 }
6702
6703 # bash completion helper
6704
6705 sub complete_backup_archives {
6706     my ($cmdname, $pname, $cvalue) = @_;
6707
6708     my $cfg = PVE::Storage::config();
6709
6710     my $storeid;
6711
6712     if ($cvalue =~ m/^([^:]+):/) {
6713         $storeid = $1;
6714     }
6715
6716     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6717
6718     my $res = [];
6719     foreach my $id (keys %$data) {
6720         foreach my $item (@{$data->{$id}}) {
6721             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6722             push @$res, $item->{volid} if defined($item->{volid});
6723         }
6724     }
6725
6726     return $res;
6727 }
6728
6729 my $complete_vmid_full = sub {
6730     my ($running) = @_;
6731
6732     my $idlist = vmstatus();
6733
6734     my $res = [];
6735
6736     foreach my $id (keys %$idlist) {
6737         my $d = $idlist->{$id};
6738         if (defined($running)) {
6739             next if $d->{template};
6740             next if $running && $d->{status} ne 'running';
6741             next if !$running && $d->{status} eq 'running';
6742         }
6743         push @$res, $id;
6744
6745     }
6746     return $res;
6747 };
6748
6749 sub complete_vmid {
6750     return &$complete_vmid_full();
6751 }
6752
6753 sub complete_vmid_stopped {
6754     return &$complete_vmid_full(0);
6755 }
6756
6757 sub complete_vmid_running {
6758     return &$complete_vmid_full(1);
6759 }
6760
6761 sub complete_storage {
6762
6763     my $cfg = PVE::Storage::config();
6764     my $ids = $cfg->{ids};
6765
6766     my $res = [];
6767     foreach my $sid (keys %$ids) {
6768         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6769         next if !$ids->{$sid}->{content}->{images};
6770         push @$res, $sid;
6771     }
6772
6773     return $res;
6774 }
6775
6776 sub complete_migration_storage {
6777     my ($cmd, $param, $current_value, $all_args) = @_;
6778
6779     my $targetnode = @$all_args[1];
6780
6781     my $cfg = PVE::Storage::config();
6782     my $ids = $cfg->{ids};
6783
6784     my $res = [];
6785     foreach my $sid (keys %$ids) {
6786         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
6787         next if !$ids->{$sid}->{content}->{images};
6788         push @$res, $sid;
6789     }
6790
6791     return $res;
6792 }
6793
6794 1;