PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  55 my $OVMF = {
  56     x86_64 => [
  57         "$EDK2_FW_BASE/OVMF_CODE.fd",
  58         "$EDK2_FW_BASE/OVMF_VARS.fd"
  59     ],
  60     aarch64 => [
  61         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  62         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  63     ],
  64 };
  65
  66 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  87         description => "Specifies the Qemu machine type.",
  88         type => 'string',
  89         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  90         maxLength => 40,
  91         optional => 1,
  92 });
  93
  94 #no warnings 'redefine';
  95
  96 sub cgroups_write {
  97    my ($controller, $vmid, $option, $value) = @_;
  98
  99    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 100    PVE::ProcFSTools::write_proc_entry($path, $value);
 101
 102 }
 103
 104 my $nodename_cache;
 105 sub nodename {
 106     $nodename_cache //= PVE::INotify::nodename();
 107     return $nodename_cache;
 108 }
 109
 110 my $watchdog_fmt = {
 111     model => {
 112         default_key => 1,
 113         type => 'string',
 114         enum => [qw(i6300esb ib700)],
 115         description => "Watchdog type to emulate.",
 116         default => 'i6300esb',
 117         optional => 1,
 118     },
 119     action => {
 120         type => 'string',
 121         enum => [qw(reset shutdown poweroff pause debug none)],
 122         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 123         optional => 1,
 124     },
 125 };
 126 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 127
 128 my $agent_fmt = {
 129     enabled => {
 130         description => "Enable/disable Qemu GuestAgent.",
 131         type => 'boolean',
 132         default => 0,
 133         default_key => 1,
 134     },
 135     fstrim_cloned_disks => {
 136         description => "Run fstrim after cloning/moving a disk.",
 137         type => 'boolean',
 138         optional => 1,
 139         default => 0
 140     },
 141     type => {
 142         description => "Select the agent type",
 143         type => 'string',
 144         default => 'virtio',
 145         optional => 1,
 146         enum => [qw(virtio isa)],
 147     },
 148 };
 149
 150 my $vga_fmt = {
 151     type => {
 152         description => "Select the VGA type.",
 153         type => 'string',
 154         default => 'std',
 155         optional => 1,
 156         default_key => 1,
 157         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 158     },
 159     memory => {
 160         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 161         type => 'integer',
 162         optional => 1,
 163         minimum => 4,
 164         maximum => 512,
 165     },
 166 };
 167
 168 my $ivshmem_fmt = {
 169     size => {
 170         type => 'integer',
 171         minimum => 1,
 172         description => "The size of the file in MB.",
 173     },
 174     name => {
 175         type => 'string',
 176         pattern => '[a-zA-Z0-9\-]+',
 177         optional => 1,
 178         format_description => 'string',
 179         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 180     },
 181 };
 182
 183 my $audio_fmt = {
 184     device => {
 185         type => 'string',
 186         enum => [qw(ich9-intel-hda intel-hda AC97)],
 187         description =>  "Configure an audio device."
 188     },
 189     driver =>  {
 190         type => 'string',
 191         enum => ['spice'],
 192         default => 'spice',
 193         optional => 1,
 194         description => "Driver backend for the audio device."
 195     },
 196 };
 197
 198 my $spice_enhancements_fmt = {
 199     foldersharing => {
 200         type => 'boolean',
 201         optional => 1,
 202         default => '0',
 203         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 204     },
 205     videostreaming =>  {
 206         type => 'string',
 207         enum => ['off', 'all', 'filter'],
 208         default => 'off',
 209         optional => 1,
 210         description => "Enable video streaming. Uses compression for detected video streams."
 211     },
 212 };
 213
 214 my $rng_fmt = {
 215     source => {
 216         type => 'string',
 217         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 218         default_key => 1,
 219         description => "The file on the host to gather entropy from. In most"
 220                      . " cases /dev/urandom should be preferred over /dev/random"
 221                      . " to avoid entropy-starvation issues on the host. Using"
 222                      . " urandom does *not* decrease security in any meaningful"
 223                      . " way, as it's still seeded from real entropy, and the"
 224                      . " bytes provided will most likely be mixed with real"
 225                      . " entropy on the guest as well. /dev/hwrng can be used"
 226                      . " to pass through a hardware RNG from the host.",
 227     },
 228     max_bytes => {
 229         type => 'integer',
 230         description => "Maximum bytes of entropy injected into the guest every"
 231                      . " 'period' milliseconds. Prefer a lower value when using"
 232                      . " /dev/random as source. Use 0 to disable limiting"
 233                      . " (potentially dangerous!).",
 234         optional => 1,
 235
 236         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 237         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 238         # of overwhelming the host, provided we're reading from /dev/urandom
 239         default => 1024,
 240     },
 241     period => {
 242         type => 'integer',
 243         description => "Every 'period' milliseconds the entropy-injection quota"
 244                      . " is reset, allowing the guest to retrieve another"
 245                      . " 'max_bytes' of entropy.",
 246         optional => 1,
 247         default => 1000,
 248     },
 249 };
 250
 251 my $confdesc = {
 252     onboot => {
 253         optional => 1,
 254         type => 'boolean',
 255         description => "Specifies whether a VM will be started during system bootup.",
 256         default => 0,
 257     },
 258     autostart => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Automatic restart after crash (currently ignored).",
 262         default => 0,
 263     },
 264     hotplug => {
 265         optional => 1,
 266         type => 'string', format => 'pve-hotplug-features',
 267         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 268         default => 'network,disk,usb',
 269     },
 270     reboot => {
 271         optional => 1,
 272         type => 'boolean',
 273         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 274         default => 1,
 275     },
 276     lock => {
 277         optional => 1,
 278         type => 'string',
 279         description => "Lock/unlock the VM.",
 280         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 281     },
 282     cpulimit => {
 283         optional => 1,
 284         type => 'number',
 285         description => "Limit of CPU usage.",
 286         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 287         minimum => 0,
 288         maximum => 128,
 289         default => 0,
 290     },
 291     cpuunits => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "CPU weight for a VM.",
 295         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 296         minimum => 2,
 297         maximum => 262144,
 298         default => 1024,
 299     },
 300     memory => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 304         minimum => 16,
 305         default => 512,
 306     },
 307     balloon => {
 308         optional => 1,
 309         type => 'integer',
 310         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 311         minimum => 0,
 312     },
 313     shares => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 317         minimum => 0,
 318         maximum => 50000,
 319         default => 1000,
 320     },
 321     keyboard => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 325                        "It should not be necessary to set it.",
 326         enum => PVE::Tools::kvmkeymaplist(),
 327         default => undef,
 328     },
 329     name => {
 330         optional => 1,
 331         type => 'string', format => 'dns-name',
 332         description => "Set a name for the VM. Only used on the configuration web interface.",
 333     },
 334     scsihw => {
 335         optional => 1,
 336         type => 'string',
 337         description => "SCSI controller model",
 338         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 339         default => 'lsi',
 340     },
 341     description => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 345     },
 346     ostype => {
 347         optional => 1,
 348         type => 'string',
 349         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 350         description => "Specify guest operating system.",
 351         verbose_description => <<EODESC,
 352 Specify guest operating system. This is used to enable special
 353 optimization/features for specific operating systems:
 354
 355 [horizontal]
 356 other;; unspecified OS
 357 wxp;; Microsoft Windows XP
 358 w2k;; Microsoft Windows 2000
 359 w2k3;; Microsoft Windows 2003
 360 w2k8;; Microsoft Windows 2008
 361 wvista;; Microsoft Windows Vista
 362 win7;; Microsoft Windows 7
 363 win8;; Microsoft Windows 8/2012/2012r2
 364 win10;; Microsoft Windows 10/2016
 365 l24;; Linux 2.4 Kernel
 366 l26;; Linux 2.6 - 5.X Kernel
 367 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 368 EODESC
 369     },
 370     boot => {
 371         optional => 1,
 372         type => 'string',
 373         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 374         pattern => '[acdn]{1,4}',
 375         default => 'cdn',
 376     },
 377     bootdisk => {
 378         optional => 1,
 379         type => 'string', format => 'pve-qm-bootdisk',
 380         description => "Enable booting from specified disk.",
 381         pattern => '(ide|sata|scsi|virtio)\d+',
 382     },
 383     smp => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPUs. Please use option -sockets instead.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     sockets => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of CPU sockets.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     cores => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "The number of cores per socket.",
 401         minimum => 1,
 402         default => 1,
 403     },
 404     numa => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Enable/disable NUMA.",
 408         default => 0,
 409     },
 410     hugepages => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Enable/disable hugepages memory.",
 414         enum => [qw(any 2 1024)],
 415     },
 416     vcpus => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "Number of hotplugged vcpus.",
 420         minimum => 1,
 421         default => 0,
 422     },
 423     acpi => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable ACPI.",
 427         default => 1,
 428     },
 429     agent => {
 430         optional => 1,
 431         description => "Enable/disable Qemu GuestAgent and its properties.",
 432         type => 'string',
 433         format => $agent_fmt,
 434     },
 435     kvm => {
 436         optional => 1,
 437         type => 'boolean',
 438         description => "Enable/disable KVM hardware virtualization.",
 439         default => 1,
 440     },
 441     tdf => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable time drift fix.",
 445         default => 0,
 446     },
 447     localtime => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 451     },
 452     freeze => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 456     },
 457     vga => {
 458         optional => 1,
 459         type => 'string', format => $vga_fmt,
 460         description => "Configure the VGA hardware.",
 461         verbose_description => "Configure the VGA Hardware. If you want to use ".
 462             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 463             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 464             "is 'std' for all OS types besides some Windows versions (XP and " .
 465             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 466             "display server. For win* OS you can select how many independent " .
 467             "displays you want, Linux guests can add displays them self.\n".
 468             "You can also run without any graphic card, using a serial device as terminal.",
 469     },
 470     watchdog => {
 471         optional => 1,
 472         type => 'string', format => 'pve-qm-watchdog',
 473         description => "Create a virtual hardware watchdog device.",
 474         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 475             " (by a guest action), the watchdog must be periodically polled " .
 476             "by an agent inside the guest or else the watchdog will reset " .
 477             "the guest (or execute the respective action specified)",
 478     },
 479     startdate => {
 480         optional => 1,
 481         type => 'string',
 482         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 483         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 484         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 485         default => 'now',
 486     },
 487     startup =>  get_standard_option('pve-startup-order'),
 488     template => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable Template.",
 492         default => 0,
 493     },
 494     args => {
 495         optional => 1,
 496         type => 'string',
 497         description => "Arbitrary arguments passed to kvm.",
 498         verbose_description => <<EODESCR,
 499 Arbitrary arguments passed to kvm, for example:
 500
 501 args: -no-reboot -no-hpet
 502
 503 NOTE: this option is for experts only.
 504 EODESCR
 505     },
 506     tablet => {
 507         optional => 1,
 508         type => 'boolean',
 509         default => 1,
 510         description => "Enable/disable the USB tablet device.",
 511         verbose_description => "Enable/disable the USB tablet device. This device is " .
 512             "usually needed to allow absolute mouse positioning with VNC. " .
 513             "Else the mouse runs out of sync with normal VNC clients. " .
 514             "If you're running lots of console-only guests on one host, " .
 515             "you may consider disabling this to save some context switches. " .
 516             "This is turned off by default if you use spice (-vga=qxl).",
 517     },
 518     migrate_speed => {
 519         optional => 1,
 520         type => 'integer',
 521         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 522         minimum => 0,
 523         default => 0,
 524     },
 525     migrate_downtime => {
 526         optional => 1,
 527         type => 'number',
 528         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 529         minimum => 0,
 530         default => 0.1,
 531     },
 532     cdrom => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-ide',
 535         typetext => '<volume>',
 536         description => "This is an alias for option -ide2",
 537     },
 538     cpu => {
 539         optional => 1,
 540         description => "Emulated CPU type.",
 541         type => 'string',
 542         format => $PVE::QemuServer::CPUConfig::cpu_fmt,
 543     },
 544     parent => get_standard_option('pve-snapshot-name', {
 545         optional => 1,
 546         description => "Parent snapshot name. This is used internally, and should not be modified.",
 547     }),
 548     snaptime => {
 549         optional => 1,
 550         description => "Timestamp for snapshots.",
 551         type => 'integer',
 552         minimum => 0,
 553     },
 554     vmstate => {
 555         optional => 1,
 556         type => 'string', format => 'pve-volume-id',
 557         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 558     },
 559     vmstatestorage => get_standard_option('pve-storage-id', {
 560         description => "Default storage for VM state volumes/files.",
 561         optional => 1,
 562     }),
 563     runningmachine => get_standard_option('pve-qemu-machine', {
 564         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 565     }),
 566     machine => get_standard_option('pve-qemu-machine'),
 567     arch => {
 568         description => "Virtual processor architecture. Defaults to the host.",
 569         optional => 1,
 570         type => 'string',
 571         enum => [qw(x86_64 aarch64)],
 572     },
 573     smbios1 => {
 574         description => "Specify SMBIOS type 1 fields.",
 575         type => 'string', format => 'pve-qm-smbios1',
 576         maxLength => 512,
 577         optional => 1,
 578     },
 579     protection => {
 580         optional => 1,
 581         type => 'boolean',
 582         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 583         default => 0,
 584     },
 585     bios => {
 586         optional => 1,
 587         type => 'string',
 588         enum => [ qw(seabios ovmf) ],
 589         description => "Select BIOS implementation.",
 590         default => 'seabios',
 591     },
 592     vmgenid => {
 593         type => 'string',
 594         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 595         format_description => 'UUID',
 596         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 597         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 598             " 128-bit integer value identifier to the guest OS. This allows to".
 599             " notify the guest operating system when the virtual machine is".
 600             " executed with a different configuration (e.g. snapshot execution".
 601             " or creation from a template). The guest operating system notices".
 602             " the change, and is then able to react as appropriate by marking".
 603             " its copies of distributed databases as dirty, re-initializing its".
 604             " random number generator, etc.\n".
 605             "Note that auto-creation only works when done throug API/CLI create".
 606             " or update methods, but not when manually editing the config file.",
 607         default => "1 (autogenerated)",
 608         optional => 1,
 609     },
 610     hookscript => {
 611         type => 'string',
 612         format => 'pve-volume-id',
 613         optional => 1,
 614         description => "Script that will be executed during various steps in the vms lifetime.",
 615     },
 616     ivshmem => {
 617         type => 'string',
 618         format => $ivshmem_fmt,
 619         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 620         optional => 1,
 621     },
 622     audio0 => {
 623         type => 'string',
 624         format => $audio_fmt,
 625         description => "Configure a audio device, useful in combination with QXL/Spice.",
 626         optional => 1
 627     },
 628     spice_enhancements => {
 629         type => 'string',
 630         format => $spice_enhancements_fmt,
 631         description => "Configure additional enhancements for SPICE.",
 632         optional => 1
 633     },
 634     tags => {
 635         type => 'string', format => 'pve-tag-list',
 636         description => 'Tags of the VM. This is only meta information.',
 637         optional => 1,
 638     },
 639     rng0 => {
 640         type => 'string',
 641         format => $rng_fmt,
 642         description => "Configure a VirtIO-based Random Number Generator.",
 643         optional => 1,
 644     },
 645 };
 646
 647 my $cicustom_fmt = {
 648     meta => {
 649         type => 'string',
 650         optional => 1,
 651         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 652         format => 'pve-volume-id',
 653         format_description => 'volume',
 654     },
 655     network => {
 656         type => 'string',
 657         optional => 1,
 658         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 659         format => 'pve-volume-id',
 660         format_description => 'volume',
 661     },
 662     user => {
 663         type => 'string',
 664         optional => 1,
 665         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 666         format => 'pve-volume-id',
 667         format_description => 'volume',
 668     },
 669 };
 670 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 671
 672 my $confdesc_cloudinit = {
 673     citype => {
 674         optional => 1,
 675         type => 'string',
 676         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 677         enum => ['configdrive2', 'nocloud'],
 678     },
 679     ciuser => {
 680         optional => 1,
 681         type => 'string',
 682         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 683     },
 684     cipassword => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 688     },
 689     cicustom => {
 690         optional => 1,
 691         type => 'string',
 692         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 693         format => 'pve-qm-cicustom',
 694     },
 695     searchdomain => {
 696         optional => 1,
 697         type => 'string',
 698         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     nameserver => {
 701         optional => 1,
 702         type => 'string', format => 'address-list',
 703         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 704     },
 705     sshkeys => {
 706         optional => 1,
 707         type => 'string',
 708         format => 'urlencoded',
 709         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 710     },
 711 };
 712
 713 # what about other qemu settings ?
 714 #cpu => 'string',
 715 #machine => 'string',
 716 #fda => 'file',
 717 #fdb => 'file',
 718 #mtdblock => 'file',
 719 #sd => 'file',
 720 #pflash => 'file',
 721 #snapshot => 'bool',
 722 #bootp => 'file',
 723 ##tftp => 'dir',
 724 ##smb => 'dir',
 725 #kernel => 'file',
 726 #append => 'string',
 727 #initrd => 'file',
 728 ##soundhw => 'string',
 729
 730 while (my ($k, $v) = each %$confdesc) {
 731     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 732 }
 733
 734 my $MAX_USB_DEVICES = 5;
 735 my $MAX_NETS = 32;
 736 my $MAX_HOSTPCI_DEVICES = 16;
 737 my $MAX_SERIAL_PORTS = 4;
 738 my $MAX_PARALLEL_PORTS = 3;
 739 my $MAX_NUMA = 8;
 740
 741 my $numa_fmt = {
 742     cpus => {
 743         type => "string",
 744         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 745         description => "CPUs accessing this NUMA node.",
 746         format_description => "id[-id];...",
 747     },
 748     memory => {
 749         type => "number",
 750         description => "Amount of memory this NUMA node provides.",
 751         optional => 1,
 752     },
 753     hostnodes => {
 754         type => "string",
 755         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 756         description => "Host NUMA nodes to use.",
 757         format_description => "id[-id];...",
 758         optional => 1,
 759     },
 760     policy => {
 761         type => 'string',
 762         enum => [qw(preferred bind interleave)],
 763         description => "NUMA allocation policy.",
 764         optional => 1,
 765     },
 766 };
 767 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 768 my $numadesc = {
 769     optional => 1,
 770     type => 'string', format => $numa_fmt,
 771     description => "NUMA topology.",
 772 };
 773 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 774
 775 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 776     $confdesc->{"numa$i"} = $numadesc;
 777 }
 778
 779 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 780                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 781                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 782 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 783
 784 my $net_fmt_bridge_descr = <<__EOD__;
 785 Bridge to attach the network device to. The Proxmox VE standard bridge
 786 is called 'vmbr0'.
 787
 788 If you do not specify a bridge, we create a kvm user (NATed) network
 789 device, which provides DHCP and DNS services. The following addresses
 790 are used:
 791
 792  10.0.2.2   Gateway
 793  10.0.2.3   DNS Server
 794  10.0.2.4   SMB Server
 795
 796 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 797 __EOD__
 798
 799 my $net_fmt = {
 800     macaddr  => get_standard_option('mac-addr', {
 801         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 802     }),
 803     model => {
 804         type => 'string',
 805         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 806         enum => $nic_model_list,
 807         default_key => 1,
 808     },
 809     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 810     bridge => {
 811         type => 'string',
 812         description => $net_fmt_bridge_descr,
 813         format_description => 'bridge',
 814         optional => 1,
 815     },
 816     queues => {
 817         type => 'integer',
 818         minimum => 0, maximum => 16,
 819         description => 'Number of packet queues to be used on the device.',
 820         optional => 1,
 821     },
 822     rate => {
 823         type => 'number',
 824         minimum => 0,
 825         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 826         optional => 1,
 827     },
 828     tag => {
 829         type => 'integer',
 830         minimum => 1, maximum => 4094,
 831         description => 'VLAN tag to apply to packets on this interface.',
 832         optional => 1,
 833     },
 834     trunks => {
 835         type => 'string',
 836         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 837         description => 'VLAN trunks to pass through this interface.',
 838         format_description => 'vlanid[;vlanid...]',
 839         optional => 1,
 840     },
 841     firewall => {
 842         type => 'boolean',
 843         description => 'Whether this interface should be protected by the firewall.',
 844         optional => 1,
 845     },
 846     link_down => {
 847         type => 'boolean',
 848         description => 'Whether this interface should be disconnected (like pulling the plug).',
 849         optional => 1,
 850     },
 851 };
 852
 853 my $netdesc = {
 854     optional => 1,
 855     type => 'string', format => $net_fmt,
 856     description => "Specify network devices.",
 857 };
 858
 859 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 860
 861 my $ipconfig_fmt = {
 862     ip => {
 863         type => 'string',
 864         format => 'pve-ipv4-config',
 865         format_description => 'IPv4Format/CIDR',
 866         description => 'IPv4 address in CIDR format.',
 867         optional => 1,
 868         default => 'dhcp',
 869     },
 870     gw => {
 871         type => 'string',
 872         format => 'ipv4',
 873         format_description => 'GatewayIPv4',
 874         description => 'Default gateway for IPv4 traffic.',
 875         optional => 1,
 876         requires => 'ip',
 877     },
 878     ip6 => {
 879         type => 'string',
 880         format => 'pve-ipv6-config',
 881         format_description => 'IPv6Format/CIDR',
 882         description => 'IPv6 address in CIDR format.',
 883         optional => 1,
 884         default => 'dhcp',
 885     },
 886     gw6 => {
 887         type => 'string',
 888         format => 'ipv6',
 889         format_description => 'GatewayIPv6',
 890         description => 'Default gateway for IPv6 traffic.',
 891         optional => 1,
 892         requires => 'ip6',
 893     },
 894 };
 895 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 896 my $ipconfigdesc = {
 897     optional => 1,
 898     type => 'string', format => 'pve-qm-ipconfig',
 899     description => <<'EODESCR',
 900 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 901
 902 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 903
 904 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 905 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 906
 907 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 908 EODESCR
 909 };
 910 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 911
 912 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 913     $confdesc->{"net$i"} = $netdesc;
 914     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 915 }
 916
 917 foreach my $key (keys %$confdesc_cloudinit) {
 918     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 919 }
 920
 921 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 922 sub verify_volume_id_or_qm_path {
 923     my ($volid, $noerr) = @_;
 924
 925     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 926         return $volid;
 927     }
 928
 929     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 930     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 931     if ($@) {
 932         return undef if $noerr;
 933         die $@;
 934     }
 935     return $volid;
 936 }
 937
 938 my $usb_fmt = {
 939     host => {
 940         default_key => 1,
 941         type => 'string', format => 'pve-qm-usb-device',
 942         format_description => 'HOSTUSBDEVICE|spice',
 943         description => <<EODESCR,
 944 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 945
 946  'bus-port(.port)*' (decimal numbers) or
 947  'vendor_id:product_id' (hexadeciaml numbers) or
 948  'spice'
 949
 950 You can use the 'lsusb -t' command to list existing usb devices.
 951
 952 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 953
 954 The value 'spice' can be used to add a usb redirection devices for spice.
 955 EODESCR
 956     },
 957     usb3 => {
 958         optional => 1,
 959         type => 'boolean',
 960         description => "Specifies whether if given host option is a USB3 device or port.",
 961         default => 0,
 962     },
 963 };
 964
 965 my $usbdesc = {
 966     optional => 1,
 967     type => 'string', format => $usb_fmt,
 968     description => "Configure an USB device (n is 0 to 4).",
 969 };
 970 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 971
 972 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
 973 my $hostpci_fmt = {
 974     host => {
 975         default_key => 1,
 976         type => 'string',
 977         pattern => qr/$PCIRE(;$PCIRE)*/,
 978         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 979         description => <<EODESCR,
 980 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 981 of PCI virtual functions of the host. HOSTPCIID syntax is:
 982
 983 'bus:dev.func' (hexadecimal numbers)
 984
 985 You can us the 'lspci' command to list existing PCI devices.
 986 EODESCR
 987     },
 988     rombar => {
 989         type => 'boolean',
 990         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 991         optional => 1,
 992         default => 1,
 993     },
 994     romfile => {
 995         type => 'string',
 996         pattern => '[^,;]+',
 997         format_description => 'string',
 998         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
 999         optional => 1,
1000     },
1001     pcie => {
1002         type => 'boolean',
1003         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1004         optional => 1,
1005         default => 0,
1006     },
1007     'x-vga' => {
1008         type => 'boolean',
1009         description =>  "Enable vfio-vga device support.",
1010         optional => 1,
1011         default => 0,
1012     },
1013     'mdev' => {
1014         type => 'string',
1015         format_description => 'string',
1016         pattern => '[^/\.:]+',
1017         optional => 1,
1018         description => <<EODESCR
1019 The type of mediated device to use.
1020 An instance of this type will be created on startup of the VM and
1021 will be cleaned up when the VM stops.
1022 EODESCR
1023     }
1024 };
1025 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1026
1027 my $hostpcidesc = {
1028         optional => 1,
1029         type => 'string', format => 'pve-qm-hostpci',
1030         description => "Map host PCI devices into guest.",
1031         verbose_description =>  <<EODESCR,
1032 Map host PCI devices into guest.
1033
1034 NOTE: This option allows direct access to host hardware. So it is no longer
1035 possible to migrate such machines - use with special care.
1036
1037 CAUTION: Experimental! User reported problems with this option.
1038 EODESCR
1039 };
1040 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1041
1042 my $serialdesc = {
1043         optional => 1,
1044         type => 'string',
1045         pattern => '(/dev/.+|socket)',
1046         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1047         verbose_description =>  <<EODESCR,
1048 Create a serial device inside the VM (n is 0 to 3), and pass through a
1049 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1050 host side (use 'qm terminal' to open a terminal connection).
1051
1052 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 my $paralleldesc= {
1059         optional => 1,
1060         type => 'string',
1061         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1062         description =>  "Map host parallel devices (n is 0 to 2).",
1063         verbose_description =>  <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $PVE::QemuServer::Drive::MAX_UNUSED_DISKS; $i++) {
1089     $confdesc->{"unused$i"} = $PVE::QemuServer::Drive::unuseddesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1093     $confdesc->{"usb$i"} = $usbdesc;
1094 }
1095
1096 my $kvm_api_version = 0;
1097
1098 sub kvm_version {
1099     return $kvm_api_version if $kvm_api_version;
1100
1101     open my $fh, '<', '/dev/kvm'
1102         or return undef;
1103
1104     # 0xae00 => KVM_GET_API_VERSION
1105     $kvm_api_version = ioctl($fh, 0xae00, 0);
1106
1107     return $kvm_api_version;
1108 }
1109
1110 my $kvm_user_version = {};
1111 my $kvm_mtime = {};
1112
1113 sub kvm_user_version {
1114     my ($binary) = @_;
1115
1116     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1117     my $st = stat($binary);
1118
1119     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1120     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1121         $cachedmtime == $st->mtime;
1122
1123     $kvm_user_version->{$binary} = 'unknown';
1124     $kvm_mtime->{$binary} = $st->mtime;
1125
1126     my $code = sub {
1127         my $line = shift;
1128         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1129             $kvm_user_version->{$binary} = $2;
1130         }
1131     };
1132
1133     eval { run_command([$binary, '--version'], outfunc => $code); };
1134     warn $@ if $@;
1135
1136     return $kvm_user_version->{$binary};
1137
1138 }
1139
1140 sub kernel_has_vhost_net {
1141     return -c '/dev/vhost-net';
1142 }
1143
1144 sub option_exists {
1145     my $key = shift;
1146     return defined($confdesc->{$key});
1147 }
1148
1149 my $cdrom_path;
1150 sub get_cdrom_path {
1151
1152     return  $cdrom_path if $cdrom_path;
1153
1154     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1155     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1156     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1157 }
1158
1159 sub get_iso_path {
1160     my ($storecfg, $vmid, $cdrom) = @_;
1161
1162     if ($cdrom eq 'cdrom') {
1163         return get_cdrom_path();
1164     } elsif ($cdrom eq 'none') {
1165         return '';
1166     } elsif ($cdrom =~ m|^/|) {
1167         return $cdrom;
1168     } else {
1169         return PVE::Storage::path($storecfg, $cdrom);
1170     }
1171 }
1172
1173 # try to convert old style file names to volume IDs
1174 sub filename_to_volume_id {
1175     my ($vmid, $file, $media) = @_;
1176
1177      if (!($file eq 'none' || $file eq 'cdrom' ||
1178           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1179
1180         return undef if $file =~ m|/|;
1181
1182         if ($media && $media eq 'cdrom') {
1183             $file = "local:iso/$file";
1184         } else {
1185             $file = "local:$vmid/$file";
1186         }
1187     }
1188
1189     return $file;
1190 }
1191
1192 sub verify_media_type {
1193     my ($opt, $vtype, $media) = @_;
1194
1195     return if !$media;
1196
1197     my $etype;
1198     if ($media eq 'disk') {
1199         $etype = 'images';
1200     } elsif ($media eq 'cdrom') {
1201         $etype = 'iso';
1202     } else {
1203         die "internal error";
1204     }
1205
1206     return if ($vtype eq $etype);
1207
1208     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1209 }
1210
1211 sub cleanup_drive_path {
1212     my ($opt, $storecfg, $drive) = @_;
1213
1214     # try to convert filesystem paths to volume IDs
1215
1216     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1217         ($drive->{file} !~ m|^/dev/.+|) &&
1218         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1219         ($drive->{file} !~ m/^\d+$/)) {
1220         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1221         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1222         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1223         verify_media_type($opt, $vtype, $drive->{media});
1224         $drive->{file} = $volid;
1225     }
1226
1227     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1228 }
1229
1230 sub parse_hotplug_features {
1231     my ($data) = @_;
1232
1233     my $res = {};
1234
1235     return $res if $data eq '0';
1236
1237     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1238
1239     foreach my $feature (PVE::Tools::split_list($data)) {
1240         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1241             $res->{$1} = 1;
1242         } else {
1243             die "invalid hotplug feature '$feature'\n";
1244         }
1245     }
1246     return $res;
1247 }
1248
1249 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1250 sub pve_verify_hotplug_features {
1251     my ($value, $noerr) = @_;
1252
1253     return $value if parse_hotplug_features($value);
1254
1255     return undef if $noerr;
1256
1257     die "unable to parse hotplug option\n";
1258 }
1259
1260 sub scsi_inquiry {
1261     my($fh, $noerr) = @_;
1262
1263     my $SG_IO = 0x2285;
1264     my $SG_GET_VERSION_NUM = 0x2282;
1265
1266     my $versionbuf = "\x00" x 8;
1267     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1268     if (!$ret) {
1269         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1270         return undef;
1271     }
1272     my $version = unpack("I", $versionbuf);
1273     if ($version < 30000) {
1274         die "scsi generic interface too old\n"  if !$noerr;
1275         return undef;
1276     }
1277
1278     my $buf = "\x00" x 36;
1279     my $sensebuf = "\x00" x 8;
1280     my $cmd = pack("C x3 C x1", 0x12, 36);
1281
1282     # see /usr/include/scsi/sg.h
1283     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1284
1285     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1286                       length($sensebuf), 0, length($buf), $buf,
1287                       $cmd, $sensebuf, 6000);
1288
1289     $ret = ioctl($fh, $SG_IO, $packet);
1290     if (!$ret) {
1291         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1292         return undef;
1293     }
1294
1295     my @res = unpack($sg_io_hdr_t, $packet);
1296     if ($res[17] || $res[18]) {
1297         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1298         return undef;
1299     }
1300
1301     my $res = {};
1302     (my $byte0, my $byte1, $res->{vendor},
1303      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1304
1305     $res->{removable} = $byte1 & 128 ? 1 : 0;
1306     $res->{type} = $byte0 & 31;
1307
1308     return $res;
1309 }
1310
1311 sub path_is_scsi {
1312     my ($path) = @_;
1313
1314     my $fh = IO::File->new("+<$path") || return undef;
1315     my $res = scsi_inquiry($fh, 1);
1316     close($fh);
1317
1318     return $res;
1319 }
1320
1321 sub print_tabletdevice_full {
1322     my ($conf, $arch) = @_;
1323
1324     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1325
1326     # we use uhci for old VMs because tablet driver was buggy in older qemu
1327     my $usbbus;
1328     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1329         $usbbus = 'ehci';
1330     } else {
1331         $usbbus = 'uhci';
1332     }
1333
1334     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1335 }
1336
1337 sub print_keyboarddevice_full {
1338     my ($conf, $arch, $machine) = @_;
1339
1340     return undef if $arch ne 'aarch64';
1341
1342     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1343 }
1344
1345 sub print_drivedevice_full {
1346     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1347
1348     my $device = '';
1349     my $maxdev = 0;
1350
1351     if ($drive->{interface} eq 'virtio') {
1352         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1353         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1354         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1355     } elsif ($drive->{interface} eq 'scsi') {
1356
1357         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1358         my $unit = $drive->{index} % $maxdev;
1359         my $devicetype = 'hd';
1360         my $path = '';
1361         if (drive_is_cdrom($drive)) {
1362             $devicetype = 'cd';
1363         } else {
1364             if ($drive->{file} =~ m|^/|) {
1365                 $path = $drive->{file};
1366                 if (my $info = path_is_scsi($path)) {
1367                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1368                         $devicetype = 'block';
1369                     } elsif ($info->{type} == 1) { # tape
1370                         $devicetype = 'generic';
1371                     }
1372                 }
1373             } else {
1374                  $path = PVE::Storage::path($storecfg, $drive->{file});
1375             }
1376
1377             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1378             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1379             if ($path =~ m/^iscsi\:\/\// &&
1380                !min_version($version, 4, 1)) {
1381                 $devicetype = 'generic';
1382             }
1383         }
1384
1385         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1386             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1387         } else {
1388             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1389         }
1390
1391         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1392             $device .= ",rotation_rate=1";
1393         }
1394         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1395
1396     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1397         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1398         my $controller = int($drive->{index} / $maxdev);
1399         my $unit = $drive->{index} % $maxdev;
1400         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1401
1402         $device = "ide-$devicetype";
1403         if ($drive->{interface} eq 'ide') {
1404             $device .= ",bus=ide.$controller,unit=$unit";
1405         } else {
1406             $device .= ",bus=ahci$controller.$unit";
1407         }
1408         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1409
1410         if ($devicetype eq 'hd') {
1411             if (my $model = $drive->{model}) {
1412                 $model = URI::Escape::uri_unescape($model);
1413                 $device .= ",model=$model";
1414             }
1415             if ($drive->{ssd}) {
1416                 $device .= ",rotation_rate=1";
1417             }
1418         }
1419         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1420     } elsif ($drive->{interface} eq 'usb') {
1421         die "implement me";
1422         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1423     } else {
1424         die "unsupported interface type";
1425     }
1426
1427     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1428
1429     if (my $serial = $drive->{serial}) {
1430         $serial = URI::Escape::uri_unescape($serial);
1431         $device .= ",serial=$serial";
1432     }
1433
1434
1435     return $device;
1436 }
1437
1438 sub get_initiator_name {
1439     my $initiator;
1440
1441     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1442     while (defined(my $line = <$fh>)) {
1443         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1444         $initiator = $1;
1445         last;
1446     }
1447     $fh->close();
1448
1449     return $initiator;
1450 }
1451
1452 sub print_drive_commandline_full {
1453     my ($storecfg, $vmid, $drive) = @_;
1454
1455     my $path;
1456     my $volid = $drive->{file};
1457     my $format;
1458
1459     if (drive_is_cdrom($drive)) {
1460         $path = get_iso_path($storecfg, $vmid, $volid);
1461     } else {
1462         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1463         if ($storeid) {
1464             $path = PVE::Storage::path($storecfg, $volid);
1465             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1466             $format = qemu_img_format($scfg, $volname);
1467         } else {
1468             $path = $volid;
1469             $format = "raw";
1470         }
1471    }
1472
1473     my $opts = '';
1474     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1475     foreach my $o (@qemu_drive_options) {
1476         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1477     }
1478
1479     # snapshot only accepts on|off
1480     if (defined($drive->{snapshot})) {
1481         my $v = $drive->{snapshot} ? 'on' : 'off';
1482         $opts .= ",snapshot=$v";
1483     }
1484
1485     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1486         my ($dir, $qmpname) = @$type;
1487         if (my $v = $drive->{"mbps$dir"}) {
1488             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1489         }
1490         if (my $v = $drive->{"mbps${dir}_max"}) {
1491             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1492         }
1493         if (my $v = $drive->{"bps${dir}_max_length"}) {
1494             $opts .= ",throttling.bps$qmpname-max-length=$v";
1495         }
1496         if (my $v = $drive->{"iops${dir}"}) {
1497             $opts .= ",throttling.iops$qmpname=$v";
1498         }
1499         if (my $v = $drive->{"iops${dir}_max"}) {
1500             $opts .= ",throttling.iops$qmpname-max=$v";
1501         }
1502         if (my $v = $drive->{"iops${dir}_max_length"}) {
1503             $opts .= ",throttling.iops$qmpname-max-length=$v";
1504         }
1505     }
1506
1507     $opts .= ",format=$format" if $format && !$drive->{format};
1508
1509     my $cache_direct = 0;
1510
1511     if (my $cache = $drive->{cache}) {
1512         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1513     } elsif (!drive_is_cdrom($drive)) {
1514         $opts .= ",cache=none";
1515         $cache_direct = 1;
1516     }
1517
1518     # aio native works only with O_DIRECT
1519     if (!$drive->{aio}) {
1520         if($cache_direct) {
1521             $opts .= ",aio=native";
1522         } else {
1523             $opts .= ",aio=threads";
1524         }
1525     }
1526
1527     if (!drive_is_cdrom($drive)) {
1528         my $detectzeroes;
1529         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1530             $detectzeroes = 'off';
1531         } elsif ($drive->{discard}) {
1532             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1533         } else {
1534             # This used to be our default with discard not being specified:
1535             $detectzeroes = 'on';
1536         }
1537         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1538     }
1539
1540     my $pathinfo = $path ? "file=$path," : '';
1541
1542     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1543 }
1544
1545 sub print_netdevice_full {
1546     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1547
1548     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1549
1550     my $device = $net->{model};
1551     if ($net->{model} eq 'virtio') {
1552          $device = 'virtio-net-pci';
1553      };
1554
1555     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1556     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1557     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1558         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1559         my $vectors = $net->{queues} * 2 + 2;
1560         $tmpstr .= ",vectors=$vectors,mq=on";
1561     }
1562     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1563
1564     if ($use_old_bios_files) {
1565         my $romfile;
1566         if ($device eq 'virtio-net-pci') {
1567             $romfile = 'pxe-virtio.rom';
1568         } elsif ($device eq 'e1000') {
1569             $romfile = 'pxe-e1000.rom';
1570         } elsif ($device eq 'ne2k') {
1571             $romfile = 'pxe-ne2k_pci.rom';
1572         } elsif ($device eq 'pcnet') {
1573             $romfile = 'pxe-pcnet.rom';
1574         } elsif ($device eq 'rtl8139') {
1575             $romfile = 'pxe-rtl8139.rom';
1576         }
1577         $tmpstr .= ",romfile=$romfile" if $romfile;
1578     }
1579
1580     return $tmpstr;
1581 }
1582
1583 sub print_netdev_full {
1584     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1585
1586     my $i = '';
1587     if ($netid =~ m/^net(\d+)$/) {
1588         $i = int($1);
1589     }
1590
1591     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1592
1593     my $ifname = "tap${vmid}i$i";
1594
1595     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1596     die "interface name '$ifname' is too long (max 15 character)\n"
1597         if length($ifname) >= 16;
1598
1599     my $vhostparam = '';
1600     if (is_native($arch)) {
1601         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1602     }
1603
1604     my $vmname = $conf->{name} || "vm$vmid";
1605
1606     my $netdev = "";
1607     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1608
1609     if ($net->{bridge}) {
1610         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1611     } else {
1612         $netdev = "type=user,id=$netid,hostname=$vmname";
1613     }
1614
1615     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1616
1617     return $netdev;
1618 }
1619
1620 my $vga_map = {
1621     'cirrus' => 'cirrus-vga',
1622     'std' => 'VGA',
1623     'vmware' => 'vmware-svga',
1624     'virtio' => 'virtio-vga',
1625 };
1626
1627 sub print_vga_device {
1628     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1629
1630     my $type = $vga_map->{$vga->{type}};
1631     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1632         $type = 'virtio-gpu';
1633     }
1634     my $vgamem_mb = $vga->{memory};
1635
1636     my $max_outputs = '';
1637     if ($qxlnum) {
1638         $type = $id ? 'qxl' : 'qxl-vga';
1639
1640         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1641             # set max outputs so linux can have up to 4 qxl displays with one device
1642             if (min_version($machine_version, 4, 1)) {
1643                 $max_outputs = ",max_outputs=4";
1644             }
1645         }
1646     }
1647
1648     die "no devicetype for $vga->{type}\n" if !$type;
1649
1650     my $memory = "";
1651     if ($vgamem_mb) {
1652         if ($vga->{type} eq 'virtio') {
1653             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1654             $memory = ",max_hostmem=$bytes";
1655         } elsif ($qxlnum) {
1656             # from https://www.spice-space.org/multiple-monitors.html
1657             $memory = ",vgamem_mb=$vga->{memory}";
1658             my $ram = $vgamem_mb * 4;
1659             my $vram = $vgamem_mb * 2;
1660             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1661         } else {
1662             $memory = ",vgamem_mb=$vga->{memory}";
1663         }
1664     } elsif ($qxlnum && $id) {
1665         $memory = ",ram_size=67108864,vram_size=33554432";
1666     }
1667
1668     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1669     my $vgaid = "vga" . ($id // '');
1670     my $pciaddr;
1671
1672     if ($q35 && $vgaid eq 'vga') {
1673         # the first display uses pcie.0 bus on q35 machines
1674         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1675     } else {
1676         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1677     }
1678
1679     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1680 }
1681
1682 sub parse_number_sets {
1683     my ($set) = @_;
1684     my $res = [];
1685     foreach my $part (split(/;/, $set)) {
1686         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1687             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1688             push @$res, [ $1, $2 ];
1689         } else {
1690             die "invalid range: $part\n";
1691         }
1692     }
1693     return $res;
1694 }
1695
1696 sub parse_numa {
1697     my ($data) = @_;
1698
1699     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1700     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1701     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1702     return $res;
1703 }
1704
1705 sub parse_hostpci {
1706     my ($value) = @_;
1707
1708     return undef if !$value;
1709
1710     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1711
1712     my @idlist = split(/;/, $res->{host});
1713     delete $res->{host};
1714     foreach my $id (@idlist) {
1715         my $devs = PVE::SysFSTools::lspci($id);
1716         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1717         push @{$res->{pciid}}, @$devs;
1718     }
1719     return $res;
1720 }
1721
1722 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1723 sub parse_net {
1724     my ($data) = @_;
1725
1726     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1727     if ($@) {
1728         warn $@;
1729         return undef;
1730     }
1731     if (!defined($res->{macaddr})) {
1732         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1733         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1734     }
1735     return $res;
1736 }
1737
1738 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1739 sub parse_ipconfig {
1740     my ($data) = @_;
1741
1742     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1743     if ($@) {
1744         warn $@;
1745         return undef;
1746     }
1747
1748     if ($res->{gw} && !$res->{ip}) {
1749         warn 'gateway specified without specifying an IP address';
1750         return undef;
1751     }
1752     if ($res->{gw6} && !$res->{ip6}) {
1753         warn 'IPv6 gateway specified without specifying an IPv6 address';
1754         return undef;
1755     }
1756     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1757         warn 'gateway specified together with DHCP';
1758         return undef;
1759     }
1760     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1761         # gw6 + auto/dhcp
1762         warn "IPv6 gateway specified together with $res->{ip6} address";
1763         return undef;
1764     }
1765
1766     if (!$res->{ip} && !$res->{ip6}) {
1767         return { ip => 'dhcp', ip6 => 'dhcp' };
1768     }
1769
1770     return $res;
1771 }
1772
1773 sub print_net {
1774     my $net = shift;
1775
1776     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1777 }
1778
1779 sub add_random_macs {
1780     my ($settings) = @_;
1781
1782     foreach my $opt (keys %$settings) {
1783         next if $opt !~ m/^net(\d+)$/;
1784         my $net = parse_net($settings->{$opt});
1785         next if !$net;
1786         $settings->{$opt} = print_net($net);
1787     }
1788 }
1789
1790 sub vm_is_volid_owner {
1791     my ($storecfg, $vmid, $volid) = @_;
1792
1793     if ($volid !~  m|^/|) {
1794         my ($path, $owner);
1795         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1796         if ($owner && ($owner == $vmid)) {
1797             return 1;
1798         }
1799     }
1800
1801     return undef;
1802 }
1803
1804 sub vmconfig_register_unused_drive {
1805     my ($storecfg, $vmid, $conf, $drive) = @_;
1806
1807     if (drive_is_cloudinit($drive)) {
1808         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1809         warn $@ if $@;
1810     } elsif (!drive_is_cdrom($drive)) {
1811         my $volid = $drive->{file};
1812         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1813             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1814         }
1815     }
1816 }
1817
1818 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1819 my $smbios1_fmt = {
1820     uuid => {
1821         type => 'string',
1822         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1823         format_description => 'UUID',
1824         description => "Set SMBIOS1 UUID.",
1825         optional => 1,
1826     },
1827     version => {
1828         type => 'string',
1829         pattern => '[A-Za-z0-9+\/]+={0,2}',
1830         format_description => 'Base64 encoded string',
1831         description => "Set SMBIOS1 version.",
1832         optional => 1,
1833     },
1834     serial => {
1835         type => 'string',
1836         pattern => '[A-Za-z0-9+\/]+={0,2}',
1837         format_description => 'Base64 encoded string',
1838         description => "Set SMBIOS1 serial number.",
1839         optional => 1,
1840     },
1841     manufacturer => {
1842         type => 'string',
1843         pattern => '[A-Za-z0-9+\/]+={0,2}',
1844         format_description => 'Base64 encoded string',
1845         description => "Set SMBIOS1 manufacturer.",
1846         optional => 1,
1847     },
1848     product => {
1849         type => 'string',
1850         pattern => '[A-Za-z0-9+\/]+={0,2}',
1851         format_description => 'Base64 encoded string',
1852         description => "Set SMBIOS1 product ID.",
1853         optional => 1,
1854     },
1855     sku => {
1856         type => 'string',
1857         pattern => '[A-Za-z0-9+\/]+={0,2}',
1858         format_description => 'Base64 encoded string',
1859         description => "Set SMBIOS1 SKU string.",
1860         optional => 1,
1861     },
1862     family => {
1863         type => 'string',
1864         pattern => '[A-Za-z0-9+\/]+={0,2}',
1865         format_description => 'Base64 encoded string',
1866         description => "Set SMBIOS1 family string.",
1867         optional => 1,
1868     },
1869     base64 => {
1870         type => 'boolean',
1871         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1872         optional => 1,
1873     },
1874 };
1875
1876 sub parse_smbios1 {
1877     my ($data) = @_;
1878
1879     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1880     warn $@ if $@;
1881     return $res;
1882 }
1883
1884 sub print_smbios1 {
1885     my ($smbios1) = @_;
1886     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1887 }
1888
1889 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1890
1891 sub parse_watchdog {
1892     my ($value) = @_;
1893
1894     return undef if !$value;
1895
1896     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1897     warn $@ if $@;
1898     return $res;
1899 }
1900
1901 sub parse_guest_agent {
1902     my ($value) = @_;
1903
1904     return {} if !defined($value->{agent});
1905
1906     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1907     warn $@ if $@;
1908
1909     # if the agent is disabled ignore the other potentially set properties
1910     return {} if !$res->{enabled};
1911     return $res;
1912 }
1913
1914 sub parse_vga {
1915     my ($value) = @_;
1916
1917     return {} if !$value;
1918     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1919     warn $@ if $@;
1920     return $res;
1921 }
1922
1923 sub parse_rng {
1924     my ($value) = @_;
1925
1926     return undef if !$value;
1927
1928     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1929     warn $@ if $@;
1930     return $res;
1931 }
1932
1933 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1934 sub verify_usb_device {
1935     my ($value, $noerr) = @_;
1936
1937     return $value if parse_usb_device($value);
1938
1939     return undef if $noerr;
1940
1941     die "unable to parse usb device\n";
1942 }
1943
1944 # add JSON properties for create and set function
1945 sub json_config_properties {
1946     my $prop = shift;
1947
1948     foreach my $opt (keys %$confdesc) {
1949         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1950         $prop->{$opt} = $confdesc->{$opt};
1951     }
1952
1953     return $prop;
1954 }
1955
1956 # return copy of $confdesc_cloudinit to generate documentation
1957 sub cloudinit_config_properties {
1958
1959     return dclone($confdesc_cloudinit);
1960 }
1961
1962 sub check_type {
1963     my ($key, $value) = @_;
1964
1965     die "unknown setting '$key'\n" if !$confdesc->{$key};
1966
1967     my $type = $confdesc->{$key}->{type};
1968
1969     if (!defined($value)) {
1970         die "got undefined value\n";
1971     }
1972
1973     if ($value =~ m/[\n\r]/) {
1974         die "property contains a line feed\n";
1975     }
1976
1977     if ($type eq 'boolean') {
1978         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1979         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1980         die "type check ('boolean') failed - got '$value'\n";
1981     } elsif ($type eq 'integer') {
1982         return int($1) if $value =~ m/^(\d+)$/;
1983         die "type check ('integer') failed - got '$value'\n";
1984     } elsif ($type eq 'number') {
1985         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1986         die "type check ('number') failed - got '$value'\n";
1987     } elsif ($type eq 'string') {
1988         if (my $fmt = $confdesc->{$key}->{format}) {
1989             PVE::JSONSchema::check_format($fmt, $value);
1990             return $value;
1991         }
1992         $value =~ s/^\"(.*)\"$/$1/;
1993         return $value;
1994     } else {
1995         die "internal error"
1996     }
1997 }
1998
1999 sub destroy_vm {
2000     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2001
2002     my $conf = PVE::QemuConfig->load_config($vmid);
2003
2004     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2005
2006     if ($conf->{template}) {
2007         # check if any base image is still used by a linked clone
2008         foreach_drive($conf, sub {
2009                 my ($ds, $drive) = @_;
2010                 return if drive_is_cdrom($drive);
2011
2012                 my $volid = $drive->{file};
2013                 return if !$volid || $volid =~ m|^/|;
2014
2015                 die "base volume '$volid' is still in use by linked cloned\n"
2016                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2017
2018         });
2019     }
2020
2021     # only remove disks owned by this VM
2022     foreach_drive($conf, sub {
2023         my ($ds, $drive) = @_;
2024         return if drive_is_cdrom($drive, 1);
2025
2026         my $volid = $drive->{file};
2027         return if !$volid || $volid =~ m|^/|;
2028
2029         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2030         return if !$path || !$owner || ($owner != $vmid);
2031
2032         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2033         warn "Could not remove disk '$volid', check manually: $@" if $@;
2034     });
2035
2036     # also remove unused disk
2037     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2038     PVE::Storage::foreach_volid($vmdisks, sub {
2039         my ($volid, $sid, $volname, $d) = @_;
2040         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2041         warn $@ if $@;
2042     });
2043
2044     if (defined $replacement_conf) {
2045         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2046     } else {
2047         PVE::QemuConfig->destroy_config($vmid);
2048     }
2049 }
2050
2051 sub parse_vm_config {
2052     my ($filename, $raw) = @_;
2053
2054     return undef if !defined($raw);
2055
2056     my $res = {
2057         digest => Digest::SHA::sha1_hex($raw),
2058         snapshots => {},
2059         pending => {},
2060     };
2061
2062     $filename =~ m|/qemu-server/(\d+)\.conf$|
2063         || die "got strange filename '$filename'";
2064
2065     my $vmid = $1;
2066
2067     my $conf = $res;
2068     my $descr;
2069     my $section = '';
2070
2071     my @lines = split(/\n/, $raw);
2072     foreach my $line (@lines) {
2073         next if $line =~ m/^\s*$/;
2074
2075         if ($line =~ m/^\[PENDING\]\s*$/i) {
2076             $section = 'pending';
2077             if (defined($descr)) {
2078                 $descr =~ s/\s+$//;
2079                 $conf->{description} = $descr;
2080             }
2081             $descr = undef;
2082             $conf = $res->{$section} = {};
2083             next;
2084
2085         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2086             $section = $1;
2087             if (defined($descr)) {
2088                 $descr =~ s/\s+$//;
2089                 $conf->{description} = $descr;
2090             }
2091             $descr = undef;
2092             $conf = $res->{snapshots}->{$section} = {};
2093             next;
2094         }
2095
2096         if ($line =~ m/^\#(.*)\s*$/) {
2097             $descr = '' if !defined($descr);
2098             $descr .= PVE::Tools::decode_text($1) . "\n";
2099             next;
2100         }
2101
2102         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2103             $descr = '' if !defined($descr);
2104             $descr .= PVE::Tools::decode_text($2);
2105         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2106             $conf->{snapstate} = $1;
2107         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2108             my $key = $1;
2109             my $value = $2;
2110             $conf->{$key} = $value;
2111         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2112             my $value = $1;
2113             if ($section eq 'pending') {
2114                 $conf->{delete} = $value; # we parse this later
2115             } else {
2116                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2117             }
2118         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2119             my $key = $1;
2120             my $value = $2;
2121             eval { $value = check_type($key, $value); };
2122             if ($@) {
2123                 warn "vm $vmid - unable to parse value of '$key' - $@";
2124             } else {
2125                 $key = 'ide2' if $key eq 'cdrom';
2126                 my $fmt = $confdesc->{$key}->{format};
2127                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2128                     my $v = parse_drive($key, $value);
2129                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2130                         $v->{file} = $volid;
2131                         $value = print_drive($v);
2132                     } else {
2133                         warn "vm $vmid - unable to parse value of '$key'\n";
2134                         next;
2135                     }
2136                 }
2137
2138                 $conf->{$key} = $value;
2139             }
2140         }
2141     }
2142
2143     if (defined($descr)) {
2144         $descr =~ s/\s+$//;
2145         $conf->{description} = $descr;
2146     }
2147     delete $res->{snapstate}; # just to be sure
2148
2149     return $res;
2150 }
2151
2152 sub write_vm_config {
2153     my ($filename, $conf) = @_;
2154
2155     delete $conf->{snapstate}; # just to be sure
2156
2157     if ($conf->{cdrom}) {
2158         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2159         $conf->{ide2} = $conf->{cdrom};
2160         delete $conf->{cdrom};
2161     }
2162
2163     # we do not use 'smp' any longer
2164     if ($conf->{sockets}) {
2165         delete $conf->{smp};
2166     } elsif ($conf->{smp}) {
2167         $conf->{sockets} = $conf->{smp};
2168         delete $conf->{cores};
2169         delete $conf->{smp};
2170     }
2171
2172     my $used_volids = {};
2173
2174     my $cleanup_config = sub {
2175         my ($cref, $pending, $snapname) = @_;
2176
2177         foreach my $key (keys %$cref) {
2178             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2179                 $key eq 'snapstate' || $key eq 'pending';
2180             my $value = $cref->{$key};
2181             if ($key eq 'delete') {
2182                 die "propertry 'delete' is only allowed in [PENDING]\n"
2183                     if !$pending;
2184                 # fixme: check syntax?
2185                 next;
2186             }
2187             eval { $value = check_type($key, $value); };
2188             die "unable to parse value of '$key' - $@" if $@;
2189
2190             $cref->{$key} = $value;
2191
2192             if (!$snapname && is_valid_drivename($key)) {
2193                 my $drive = parse_drive($key, $value);
2194                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2195             }
2196         }
2197     };
2198
2199     &$cleanup_config($conf);
2200
2201     &$cleanup_config($conf->{pending}, 1);
2202
2203     foreach my $snapname (keys %{$conf->{snapshots}}) {
2204         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2205         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2206     }
2207
2208     # remove 'unusedX' settings if we re-add a volume
2209     foreach my $key (keys %$conf) {
2210         my $value = $conf->{$key};
2211         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2212             delete $conf->{$key};
2213         }
2214     }
2215
2216     my $generate_raw_config = sub {
2217         my ($conf, $pending) = @_;
2218
2219         my $raw = '';
2220
2221         # add description as comment to top of file
2222         if (defined(my $descr = $conf->{description})) {
2223             if ($descr) {
2224                 foreach my $cl (split(/\n/, $descr)) {
2225                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2226                 }
2227             } else {
2228                 $raw .= "#\n" if $pending;
2229             }
2230         }
2231
2232         foreach my $key (sort keys %$conf) {
2233             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2234             $raw .= "$key: $conf->{$key}\n";
2235         }
2236         return $raw;
2237     };
2238
2239     my $raw = &$generate_raw_config($conf);
2240
2241     if (scalar(keys %{$conf->{pending}})){
2242         $raw .= "\n[PENDING]\n";
2243         $raw .= &$generate_raw_config($conf->{pending}, 1);
2244     }
2245
2246     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2247         $raw .= "\n[$snapname]\n";
2248         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2249     }
2250
2251     return $raw;
2252 }
2253
2254 sub load_defaults {
2255
2256     my $res = {};
2257
2258     # we use static defaults from our JSON schema configuration
2259     foreach my $key (keys %$confdesc) {
2260         if (defined(my $default = $confdesc->{$key}->{default})) {
2261             $res->{$key} = $default;
2262         }
2263     }
2264
2265     return $res;
2266 }
2267
2268 sub config_list {
2269     my $vmlist = PVE::Cluster::get_vmlist();
2270     my $res = {};
2271     return $res if !$vmlist || !$vmlist->{ids};
2272     my $ids = $vmlist->{ids};
2273     my $nodename = nodename();
2274
2275     foreach my $vmid (keys %$ids) {
2276         my $d = $ids->{$vmid};
2277         next if !$d->{node} || $d->{node} ne $nodename;
2278         next if !$d->{type} || $d->{type} ne 'qemu';
2279         $res->{$vmid}->{exists} = 1;
2280     }
2281     return $res;
2282 }
2283
2284 # test if VM uses local resources (to prevent migration)
2285 sub check_local_resources {
2286     my ($conf, $noerr) = @_;
2287
2288     my @loc_res = ();
2289
2290     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2291     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2292
2293     push @loc_res, "ivshmem" if $conf->{ivshmem};
2294
2295     foreach my $k (keys %$conf) {
2296         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2297         # sockets are safe: they will recreated be on the target side post-migrate
2298         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2299         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2300     }
2301
2302     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2303
2304     return \@loc_res;
2305 }
2306
2307 # check if used storages are available on all nodes (use by migrate)
2308 sub check_storage_availability {
2309     my ($storecfg, $conf, $node) = @_;
2310
2311     foreach_drive($conf, sub {
2312         my ($ds, $drive) = @_;
2313
2314         my $volid = $drive->{file};
2315         return if !$volid;
2316
2317         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2318         return if !$sid;
2319
2320         # check if storage is available on both nodes
2321         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2322         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2323    });
2324 }
2325
2326 # list nodes where all VM images are available (used by has_feature API)
2327 sub shared_nodes {
2328     my ($conf, $storecfg) = @_;
2329
2330     my $nodelist = PVE::Cluster::get_nodelist();
2331     my $nodehash = { map { $_ => 1 } @$nodelist };
2332     my $nodename = nodename();
2333
2334     foreach_drive($conf, sub {
2335         my ($ds, $drive) = @_;
2336
2337         my $volid = $drive->{file};
2338         return if !$volid;
2339
2340         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2341         if ($storeid) {
2342             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2343             if ($scfg->{disable}) {
2344                 $nodehash = {};
2345             } elsif (my $avail = $scfg->{nodes}) {
2346                 foreach my $node (keys %$nodehash) {
2347                     delete $nodehash->{$node} if !$avail->{$node};
2348                 }
2349             } elsif (!$scfg->{shared}) {
2350                 foreach my $node (keys %$nodehash) {
2351                     delete $nodehash->{$node} if $node ne $nodename
2352                 }
2353             }
2354         }
2355     });
2356
2357     return $nodehash
2358 }
2359
2360 sub check_local_storage_availability {
2361     my ($conf, $storecfg) = @_;
2362
2363     my $nodelist = PVE::Cluster::get_nodelist();
2364     my $nodehash = { map { $_ => {} } @$nodelist };
2365
2366     foreach_drive($conf, sub {
2367         my ($ds, $drive) = @_;
2368
2369         my $volid = $drive->{file};
2370         return if !$volid;
2371
2372         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2373         if ($storeid) {
2374             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2375
2376             if ($scfg->{disable}) {
2377                 foreach my $node (keys %$nodehash) {
2378                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2379                 }
2380             } elsif (my $avail = $scfg->{nodes}) {
2381                 foreach my $node (keys %$nodehash) {
2382                     if (!$avail->{$node}) {
2383                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2384                     }
2385                 }
2386             }
2387         }
2388     });
2389
2390     foreach my $node (values %$nodehash) {
2391         if (my $unavail = $node->{unavailable_storages}) {
2392             $node->{unavailable_storages} = [ sort keys %$unavail ];
2393         }
2394     }
2395
2396     return $nodehash
2397 }
2398
2399 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2400 sub check_running {
2401     my ($vmid, $nocheck, $node) = @_;
2402
2403     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2404     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2405 }
2406
2407 sub vzlist {
2408
2409     my $vzlist = config_list();
2410
2411     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2412
2413     while (defined(my $de = $fd->read)) {
2414         next if $de !~ m/^(\d+)\.pid$/;
2415         my $vmid = $1;
2416         next if !defined($vzlist->{$vmid});
2417         if (my $pid = check_running($vmid)) {
2418             $vzlist->{$vmid}->{pid} = $pid;
2419         }
2420     }
2421
2422     return $vzlist;
2423 }
2424
2425 our $vmstatus_return_properties = {
2426     vmid => get_standard_option('pve-vmid'),
2427     status => {
2428         description => "Qemu process status.",
2429         type => 'string',
2430         enum => ['stopped', 'running'],
2431     },
2432     maxmem => {
2433         description => "Maximum memory in bytes.",
2434         type => 'integer',
2435         optional => 1,
2436         renderer => 'bytes',
2437     },
2438     maxdisk => {
2439         description => "Root disk size in bytes.",
2440         type => 'integer',
2441         optional => 1,
2442         renderer => 'bytes',
2443     },
2444     name => {
2445         description => "VM name.",
2446         type => 'string',
2447         optional => 1,
2448     },
2449     qmpstatus => {
2450         description => "Qemu QMP agent status.",
2451         type => 'string',
2452         optional => 1,
2453     },
2454     pid => {
2455         description => "PID of running qemu process.",
2456         type => 'integer',
2457         optional => 1,
2458     },
2459     uptime => {
2460         description => "Uptime.",
2461         type => 'integer',
2462         optional => 1,
2463         renderer => 'duration',
2464     },
2465     cpus => {
2466         description => "Maximum usable CPUs.",
2467         type => 'number',
2468         optional => 1,
2469     },
2470     lock => {
2471         description => "The current config lock, if any.",
2472         type => 'string',
2473         optional => 1,
2474     },
2475     tags => {
2476         description  => "The current configured tags, if any",
2477         type => 'string',
2478         optional => 1,
2479     },
2480 };
2481
2482 my $last_proc_pid_stat;
2483
2484 # get VM status information
2485 # This must be fast and should not block ($full == false)
2486 # We only query KVM using QMP if $full == true (this can be slow)
2487 sub vmstatus {
2488     my ($opt_vmid, $full) = @_;
2489
2490     my $res = {};
2491
2492     my $storecfg = PVE::Storage::config();
2493
2494     my $list = vzlist();
2495     my $defaults = load_defaults();
2496
2497     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2498
2499     my $cpucount = $cpuinfo->{cpus} || 1;
2500
2501     foreach my $vmid (keys %$list) {
2502         next if $opt_vmid && ($vmid ne $opt_vmid);
2503
2504         my $conf = PVE::QemuConfig->load_config($vmid);
2505
2506         my $d = { vmid => $vmid };
2507         $d->{pid} = $list->{$vmid}->{pid};
2508
2509         # fixme: better status?
2510         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2511
2512         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2513         if (defined($size)) {
2514             $d->{disk} = 0; # no info available
2515             $d->{maxdisk} = $size;
2516         } else {
2517             $d->{disk} = 0;
2518             $d->{maxdisk} = 0;
2519         }
2520
2521         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2522             * ($conf->{cores} || $defaults->{cores});
2523         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2524         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2525
2526         $d->{name} = $conf->{name} || "VM $vmid";
2527         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2528             : $defaults->{memory}*(1024*1024);
2529
2530         if ($conf->{balloon}) {
2531             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2532             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2533                 : $defaults->{shares};
2534         }
2535
2536         $d->{uptime} = 0;
2537         $d->{cpu} = 0;
2538         $d->{mem} = 0;
2539
2540         $d->{netout} = 0;
2541         $d->{netin} = 0;
2542
2543         $d->{diskread} = 0;
2544         $d->{diskwrite} = 0;
2545
2546         $d->{template} = PVE::QemuConfig->is_template($conf);
2547
2548         $d->{serial} = 1 if conf_has_serial($conf);
2549         $d->{lock} = $conf->{lock} if $conf->{lock};
2550         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2551
2552         $res->{$vmid} = $d;
2553     }
2554
2555     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2556     foreach my $dev (keys %$netdev) {
2557         next if $dev !~ m/^tap([1-9]\d*)i/;
2558         my $vmid = $1;
2559         my $d = $res->{$vmid};
2560         next if !$d;
2561
2562         $d->{netout} += $netdev->{$dev}->{receive};
2563         $d->{netin} += $netdev->{$dev}->{transmit};
2564
2565         if ($full) {
2566             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2567             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2568         }
2569
2570     }
2571
2572     my $ctime = gettimeofday;
2573
2574     foreach my $vmid (keys %$list) {
2575
2576         my $d = $res->{$vmid};
2577         my $pid = $d->{pid};
2578         next if !$pid;
2579
2580         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2581         next if !$pstat; # not running
2582
2583         my $used = $pstat->{utime} + $pstat->{stime};
2584
2585         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2586
2587         if ($pstat->{vsize}) {
2588             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2589         }
2590
2591         my $old = $last_proc_pid_stat->{$pid};
2592         if (!$old) {
2593             $last_proc_pid_stat->{$pid} = {
2594                 time => $ctime,
2595                 used => $used,
2596                 cpu => 0,
2597             };
2598             next;
2599         }
2600
2601         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2602
2603         if ($dtime > 1000) {
2604             my $dutime = $used -  $old->{used};
2605
2606             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2607             $last_proc_pid_stat->{$pid} = {
2608                 time => $ctime,
2609                 used => $used,
2610                 cpu => $d->{cpu},
2611             };
2612         } else {
2613             $d->{cpu} = $old->{cpu};
2614         }
2615     }
2616
2617     return $res if !$full;
2618
2619     my $qmpclient = PVE::QMPClient->new();
2620
2621     my $ballooncb = sub {
2622         my ($vmid, $resp) = @_;
2623
2624         my $info = $resp->{'return'};
2625         return if !$info->{max_mem};
2626
2627         my $d = $res->{$vmid};
2628
2629         # use memory assigned to VM
2630         $d->{maxmem} = $info->{max_mem};
2631         $d->{balloon} = $info->{actual};
2632
2633         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2634             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2635             $d->{freemem} = $info->{free_mem};
2636         }
2637
2638         $d->{ballooninfo} = $info;
2639     };
2640
2641     my $blockstatscb = sub {
2642         my ($vmid, $resp) = @_;
2643         my $data = $resp->{'return'} || [];
2644         my $totalrdbytes = 0;
2645         my $totalwrbytes = 0;
2646
2647         for my $blockstat (@$data) {
2648             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2649             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2650
2651             $blockstat->{device} =~ s/drive-//;
2652             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2653         }
2654         $res->{$vmid}->{diskread} = $totalrdbytes;
2655         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2656     };
2657
2658     my $statuscb = sub {
2659         my ($vmid, $resp) = @_;
2660
2661         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2662         # this fails if ballon driver is not loaded, so this must be
2663         # the last commnand (following command are aborted if this fails).
2664         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2665
2666         my $status = 'unknown';
2667         if (!defined($status = $resp->{'return'}->{status})) {
2668             warn "unable to get VM status\n";
2669             return;
2670         }
2671
2672         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2673     };
2674
2675     foreach my $vmid (keys %$list) {
2676         next if $opt_vmid && ($vmid ne $opt_vmid);
2677         next if !$res->{$vmid}->{pid}; # not running
2678         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2679     }
2680
2681     $qmpclient->queue_execute(undef, 2);
2682
2683     foreach my $vmid (keys %$list) {
2684         next if $opt_vmid && ($vmid ne $opt_vmid);
2685         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2686     }
2687
2688     return $res;
2689 }
2690
2691 sub conf_has_serial {
2692     my ($conf) = @_;
2693
2694     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2695         if ($conf->{"serial$i"}) {
2696             return 1;
2697         }
2698     }
2699
2700     return 0;
2701 }
2702
2703 sub conf_has_audio {
2704     my ($conf, $id) = @_;
2705
2706     $id //= 0;
2707     my $audio = $conf->{"audio$id"};
2708     return undef if !defined($audio);
2709
2710     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2711     my $audiodriver = $audioproperties->{driver} // 'spice';
2712
2713     return {
2714         dev => $audioproperties->{device},
2715         dev_id => "audiodev$id",
2716         backend => $audiodriver,
2717         backend_id => "$audiodriver-backend${id}",
2718     };
2719 }
2720
2721 sub vga_conf_has_spice {
2722     my ($vga) = @_;
2723
2724     my $vgaconf = parse_vga($vga);
2725     my $vgatype = $vgaconf->{type};
2726     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2727
2728     return $1 || 1;
2729 }
2730
2731 sub is_native($) {
2732     my ($arch) = @_;
2733     return get_host_arch() eq $arch;
2734 }
2735
2736 sub get_vm_arch {
2737     my ($conf) = @_;
2738     return $conf->{arch} // get_host_arch();
2739 }
2740
2741 my $default_machines = {
2742     x86_64 => 'pc',
2743     aarch64 => 'virt',
2744 };
2745
2746 sub get_vm_machine {
2747     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2748
2749     my $machine = $forcemachine || $conf->{machine};
2750
2751     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2752         $arch //= 'x86_64';
2753         $machine ||= $default_machines->{$arch};
2754         if ($add_pve_version) {
2755             $kvmversion //= kvm_user_version();
2756             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2757             $machine .= "+pve$pvever";
2758         }
2759     }
2760
2761     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2762         # for version-pinned machines that do not include a pve-version (e.g.
2763         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2764         $machine .= '+pve0';
2765     }
2766
2767     return $machine;
2768 }
2769
2770 sub get_ovmf_files($) {
2771     my ($arch) = @_;
2772
2773     my $ovmf = $OVMF->{$arch}
2774         or die "no OVMF images known for architecture '$arch'\n";
2775
2776     return @$ovmf;
2777 }
2778
2779 my $Arch2Qemu = {
2780     aarch64 => '/usr/bin/qemu-system-aarch64',
2781     x86_64 => '/usr/bin/qemu-system-x86_64',
2782 };
2783 sub get_command_for_arch($) {
2784     my ($arch) = @_;
2785     return '/usr/bin/kvm' if is_native($arch);
2786
2787     my $cmd = $Arch2Qemu->{$arch}
2788         or die "don't know how to emulate architecture '$arch'\n";
2789     return $cmd;
2790 }
2791
2792 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2793 # to use in a QEMU command line (-cpu element), first array_intersect the result
2794 # of query_supported_ with query_understood_. This is necessary because:
2795 #
2796 # a) query_understood_ returns flags the host cannot use and
2797 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2798 #    flags, but CPU settings - with most of them being flags. Those settings
2799 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2800 #
2801 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2802 # expensive. If you need the value returned from this, you can get it much
2803 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2804 # $accel being 'kvm' or 'tcg'.
2805 #
2806 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2807 # changes, automatically populating pmxcfs.
2808 #
2809 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2810 # since kvm and tcg machines support different flags
2811 #
2812 sub query_supported_cpu_flags {
2813     my ($arch) = @_;
2814
2815     $arch //= get_host_arch();
2816     my $default_machine = $default_machines->{$arch};
2817
2818     my $flags = {};
2819
2820     # FIXME: Once this is merged, the code below should work for ARM as well:
2821     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2822     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2823         $arch eq "aarch64";
2824
2825     my $kvm_supported = defined(kvm_version());
2826     my $qemu_cmd = get_command_for_arch($arch);
2827     my $fakevmid = -1;
2828     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2829
2830     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2831     my $query_supported_run_qemu = sub {
2832         my ($kvm) = @_;
2833
2834         my $flags = {};
2835         my $cmd = [
2836             $qemu_cmd,
2837             '-machine', $default_machine,
2838             '-display', 'none',
2839             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2840             '-mon', 'chardev=qmp,mode=control',
2841             '-pidfile', $pidfile,
2842             '-S', '-daemonize'
2843         ];
2844
2845         if (!$kvm) {
2846             push @$cmd, '-accel', 'tcg';
2847         }
2848
2849         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2850         die "QEMU flag querying VM exited with code " . $rc if $rc;
2851
2852         eval {
2853             my $cmd_result = mon_cmd(
2854                 $fakevmid,
2855                 'query-cpu-model-expansion',
2856                 type => 'full',
2857                 model => { name => 'host' }
2858             );
2859
2860             my $props = $cmd_result->{model}->{props};
2861             foreach my $prop (keys %$props) {
2862                 next if $props->{$prop} ne '1';
2863                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2864                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2865                 # We only keep those with underscores, to match /proc/cpuinfo
2866                 $prop =~ s/\.|-/_/g;
2867                 $flags->{$prop} = 1;
2868             }
2869         };
2870         my $err = $@;
2871
2872         # force stop with 10 sec timeout and 'nocheck'
2873         # always stop, even if QMP failed
2874         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2875
2876         die $err if $err;
2877
2878         return [ sort keys %$flags ];
2879     };
2880
2881     # We need to query QEMU twice, since KVM and TCG have different supported flags
2882     PVE::QemuConfig->lock_config($fakevmid, sub {
2883         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2884         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2885
2886         if ($kvm_supported) {
2887             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2888             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2889         }
2890     });
2891
2892     return $flags;
2893 }
2894
2895 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2896 my $understood_cpu_flag_dir = "/usr/share/kvm";
2897 sub query_understood_cpu_flags {
2898     my $arch = get_host_arch();
2899     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2900
2901     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2902         if ! -e $filepath;
2903
2904     my $raw = file_get_contents($filepath);
2905     $raw =~ s/^\s+|\s+$//g;
2906     my @flags = split(/\s+/, $raw);
2907
2908     return \@flags;
2909 }
2910
2911 sub config_to_command {
2912     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2913
2914     my $cmd = [];
2915     my $globalFlags = [];
2916     my $machineFlags = [];
2917     my $rtcFlags = [];
2918     my $devices = [];
2919     my $pciaddr = '';
2920     my $bridges = {};
2921     my $ostype = $conf->{ostype};
2922     my $winversion = windows_version($ostype);
2923     my $kvm = $conf->{kvm};
2924     my $nodename = nodename();
2925
2926     my $arch = get_vm_arch($conf);
2927     my $kvm_binary = get_command_for_arch($arch);
2928     my $kvmver = kvm_user_version($kvm_binary);
2929
2930     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2931         $kvmver //= "undefined";
2932         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2933     }
2934
2935     my $add_pve_version = min_version($kvmver, 4, 1);
2936
2937     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2938     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2939     $kvm //= 1 if is_native($arch);
2940
2941     $machine_version =~ m/(\d+)\.(\d+)/;
2942     my ($machine_major, $machine_minor) = ($1, $2);
2943     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2944         if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
2945
2946     if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2947         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2948         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2949           . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2950     }
2951
2952     # if a specific +pve version is required for a feature, use $version_guard
2953     # instead of min_version to allow machines to be run with the minimum
2954     # required version
2955     my $required_pve_version = 0;
2956     my $version_guard = sub {
2957         my ($major, $minor, $pve) = @_;
2958         return 0 if !min_version($machine_version, $major, $minor, $pve);
2959         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2960         return 1;
2961     };
2962
2963     if ($kvm) {
2964         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2965             if !defined kvm_version();
2966     }
2967
2968     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2969     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2970     my $use_old_bios_files = undef;
2971     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2972
2973     my $cpuunits = defined($conf->{cpuunits}) ?
2974             $conf->{cpuunits} : $defaults->{cpuunits};
2975
2976     push @$cmd, $kvm_binary;
2977
2978     push @$cmd, '-id', $vmid;
2979
2980     my $vmname = $conf->{name} || "vm$vmid";
2981
2982     push @$cmd, '-name', $vmname;
2983
2984     my $use_virtio = 0;
2985
2986     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2987     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2988     push @$cmd, '-mon', "chardev=qmp,mode=control";
2989
2990     if (min_version($machine_version, 2, 12)) {
2991         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2992         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2993     }
2994
2995     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
2996
2997     push @$cmd, '-daemonize';
2998
2999     if ($conf->{smbios1}) {
3000         my $smbios_conf = parse_smbios1($conf->{smbios1});
3001         if ($smbios_conf->{base64}) {
3002             # Do not pass base64 flag to qemu
3003             delete $smbios_conf->{base64};
3004             my $smbios_string = "";
3005             foreach my $key (keys %$smbios_conf) {
3006                 my $value;
3007                 if ($key eq "uuid") {
3008                     $value = $smbios_conf->{uuid}
3009                 } else {
3010                     $value = decode_base64($smbios_conf->{$key});
3011                 }
3012                 # qemu accepts any binary data, only commas need escaping by double comma
3013                 $value =~ s/,/,,/g;
3014                 $smbios_string .= "," . $key . "=" . $value if $value;
3015             }
3016             push @$cmd, '-smbios', "type=1" . $smbios_string;
3017         } else {
3018             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3019         }
3020     }
3021
3022     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3023     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3024         die "uefi base image not found\n" if ! -f $ovmf_code;
3025
3026         my $path;
3027         my $format;
3028         if (my $efidisk = $conf->{efidisk0}) {
3029             my $d = parse_drive('efidisk0', $efidisk);
3030             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3031             $format = $d->{format};
3032             if ($storeid) {
3033                 $path = PVE::Storage::path($storecfg, $d->{file});
3034                 if (!defined($format)) {
3035                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3036                     $format = qemu_img_format($scfg, $volname);
3037                 }
3038             } else {
3039                 $path = $d->{file};
3040                 die "efidisk format must be specified\n"
3041                     if !defined($format);
3042             }
3043         } else {
3044             warn "no efidisk configured! Using temporary efivars disk.\n";
3045             $path = "/tmp/$vmid-ovmf.fd";
3046             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3047             $format = 'raw';
3048         }
3049
3050         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3051         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3052     }
3053
3054     # load q35 config
3055     if ($q35) {
3056         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3057         if (min_version($machine_version, 4, 0)) {
3058             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3059         } else {
3060             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3061         }
3062     }
3063
3064     if ($conf->{vmgenid}) {
3065         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3066     }
3067
3068     # add usb controllers
3069     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3070     push @$devices, @usbcontrollers if @usbcontrollers;
3071     my $vga = parse_vga($conf->{vga});
3072
3073     my $qxlnum = vga_conf_has_spice($conf->{vga});
3074     $vga->{type} = 'qxl' if $qxlnum;
3075
3076     if (!$vga->{type}) {
3077         if ($arch eq 'aarch64') {
3078             $vga->{type} = 'virtio';
3079         } elsif (min_version($machine_version, 2, 9)) {
3080             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3081         } else {
3082             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3083         }
3084     }
3085
3086     # enable absolute mouse coordinates (needed by vnc)
3087     my $tablet;
3088     if (defined($conf->{tablet})) {
3089         $tablet = $conf->{tablet};
3090     } else {
3091         $tablet = $defaults->{tablet};
3092         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3093         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3094     }
3095
3096     if ($tablet) {
3097         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3098         my $kbd = print_keyboarddevice_full($conf, $arch);
3099         push @$devices, '-device', $kbd if defined($kbd);
3100     }
3101
3102     my $kvm_off = 0;
3103     my $gpu_passthrough;
3104
3105     # host pci devices
3106     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3107         my $id = "hostpci$i";
3108         my $d = parse_hostpci($conf->{$id});
3109         next if !$d;
3110
3111         if (my $pcie = $d->{pcie}) {
3112             die "q35 machine model is not enabled" if !$q35;
3113             # win7 wants to have the pcie devices directly on the pcie bus
3114             # instead of in the root port
3115             if ($winversion == 7) {
3116                 $pciaddr = print_pcie_addr("${id}bus0");
3117             } else {
3118                 # add more root ports if needed, 4 are present by default
3119                 # by pve-q35 cfgs, rest added here on demand.
3120                 if ($i > 3) {
3121                     push @$devices, '-device', print_pcie_root_port($i);
3122                 }
3123                 $pciaddr = print_pcie_addr($id);
3124             }
3125         } else {
3126             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3127         }
3128
3129         my $xvga = '';
3130         if ($d->{'x-vga'}) {
3131             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3132             $kvm_off = 1;
3133             $vga->{type} = 'none' if !defined($conf->{vga});
3134             $gpu_passthrough = 1;
3135         }
3136
3137         my $pcidevices = $d->{pciid};
3138         my $multifunction = 1 if @$pcidevices > 1;
3139
3140         my $sysfspath;
3141         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3142             my $pci_id = $pcidevices->[0]->{id};
3143             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3144             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3145         } elsif ($d->{mdev}) {
3146             warn "ignoring mediated device '$id' with multifunction device\n";
3147         }
3148
3149         my $j=0;
3150         foreach my $pcidevice (@$pcidevices) {
3151             my $devicestr = "vfio-pci";
3152
3153             if ($sysfspath) {
3154                 $devicestr .= ",sysfsdev=$sysfspath";
3155             } else {
3156                 $devicestr .= ",host=$pcidevice->{id}";
3157             }
3158
3159             my $mf_addr = $multifunction ? ".$j" : '';
3160             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3161
3162             if ($j == 0) {
3163                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3164                 $devicestr .= "$xvga";
3165                 $devicestr .= ",multifunction=on" if $multifunction;
3166                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3167             }
3168
3169             push @$devices, '-device', $devicestr;
3170             $j++;
3171         }
3172     }
3173
3174     # usb devices
3175     my $usb_dev_features = {};
3176     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3177
3178     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3179     push @$devices, @usbdevices if @usbdevices;
3180     # serial devices
3181     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3182         if (my $path = $conf->{"serial$i"}) {
3183             if ($path eq 'socket') {
3184                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3185                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3186                 # On aarch64, serial0 is the UART device. Qemu only allows
3187                 # connecting UART devices via the '-serial' command line, as
3188                 # the device has a fixed slot on the hardware...
3189                 if ($arch eq 'aarch64' && $i == 0) {
3190                     push @$devices, '-serial', "chardev:serial$i";
3191                 } else {
3192                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3193                 }
3194             } else {
3195                 die "no such serial device\n" if ! -c $path;
3196                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3197                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3198             }
3199         }
3200     }
3201
3202     # parallel devices
3203     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3204         if (my $path = $conf->{"parallel$i"}) {
3205             die "no such parallel device\n" if ! -c $path;
3206             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3207             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3208             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3209         }
3210     }
3211
3212     if (my $audio = conf_has_audio($conf)) {
3213
3214         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3215
3216         my $id = $audio->{dev_id};
3217         if ($audio->{dev} eq 'AC97') {
3218             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3219         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3220             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3221             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3222             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3223         } else {
3224             die "unkown audio device '$audio->{dev}', implement me!";
3225         }
3226
3227         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3228     }
3229
3230     my $sockets = 1;
3231     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3232     $sockets = $conf->{sockets} if  $conf->{sockets};
3233
3234     my $cores = $conf->{cores} || 1;
3235
3236     my $maxcpus = $sockets * $cores;
3237
3238     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3239
3240     my $allowed_vcpus = $cpuinfo->{cpus};
3241
3242     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3243         if ($allowed_vcpus < $maxcpus);
3244
3245     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3246
3247         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3248         for (my $i = 2; $i <= $vcpus; $i++)  {
3249             my $cpustr = print_cpu_device($conf,$i);
3250             push @$cmd, '-device', $cpustr;
3251         }
3252
3253     } else {
3254
3255         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3256     }
3257     push @$cmd, '-nodefaults';
3258
3259     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3260
3261     my $bootindex_hash = {};
3262     my $i = 1;
3263     foreach my $o (split(//, $bootorder)) {
3264         $bootindex_hash->{$o} = $i*100;
3265         $i++;
3266     }
3267
3268     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3269
3270     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3271
3272     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3273
3274     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3275         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3276         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3277         push @$cmd,  '-vnc', "unix:$socket,password";
3278     } else {
3279         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3280         push @$cmd, '-nographic';
3281     }
3282
3283     # time drift fix
3284     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3285
3286     my $useLocaltime = $conf->{localtime};
3287
3288     if ($winversion >= 5) { # windows
3289         $useLocaltime = 1 if !defined($conf->{localtime});
3290
3291         # use time drift fix when acpi is enabled
3292         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3293             $tdf = 1 if !defined($conf->{tdf});
3294         }
3295     }
3296
3297     if ($winversion >= 6) {
3298         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3299         push @$cmd, '-no-hpet';
3300     }
3301
3302     push @$rtcFlags, 'driftfix=slew' if $tdf;
3303
3304     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3305         push @$rtcFlags, "base=$conf->{startdate}";
3306     } elsif ($useLocaltime) {
3307         push @$rtcFlags, 'base=localtime';
3308     }
3309
3310     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3311
3312     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3313
3314     push @$cmd, '-S' if $conf->{freeze};
3315
3316     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3317
3318     my $guest_agent = parse_guest_agent($conf);
3319
3320     if ($guest_agent->{enabled}) {
3321         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3322         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3323
3324         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3325             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3326             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3327             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3328         } elsif ($guest_agent->{type} eq 'isa') {
3329             push @$devices, '-device', "isa-serial,chardev=qga0";
3330         }
3331     }
3332
3333     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3334     if ($rng && &$version_guard(4, 1, 2)) {
3335         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3336         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3337
3338         my $limiter_str = "";
3339         if ($max_bytes) {
3340             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3341         }
3342
3343         # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3344         die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3345             if ! -e $rng->{source};
3346
3347         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3348
3349         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3350         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3351     }
3352
3353     my $spice_port;
3354
3355     if ($qxlnum) {
3356         if ($qxlnum > 1) {
3357             if ($winversion){
3358                 for(my $i = 1; $i < $qxlnum; $i++){
3359                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3360                 }
3361             } else {
3362                 # assume other OS works like Linux
3363                 my ($ram, $vram) = ("134217728", "67108864");
3364                 if ($vga->{memory}) {
3365                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3366                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3367                 }
3368                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3369                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3370             }
3371         }
3372
3373         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3374
3375         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3376         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3377         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3378
3379         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3380         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3381         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3382
3383         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3384         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3385
3386         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3387         if ($spice_enhancement->{foldersharing}) {
3388             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3389             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3390         }
3391
3392         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3393         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3394         push @$devices, '-spice', "$spice_opts";
3395     }
3396
3397     # enable balloon by default, unless explicitly disabled
3398     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3399         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3400         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3401     }
3402
3403     if ($conf->{watchdog}) {
3404         my $wdopts = parse_watchdog($conf->{watchdog});
3405         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3406         my $watchdog = $wdopts->{model} || 'i6300esb';
3407         push @$devices, '-device', "$watchdog$pciaddr";
3408         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3409     }
3410
3411     my $vollist = [];
3412     my $scsicontroller = {};
3413     my $ahcicontroller = {};
3414     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3415
3416     # Add iscsi initiator name if available
3417     if (my $initiator = get_initiator_name()) {
3418         push @$devices, '-iscsi', "initiator-name=$initiator";
3419     }
3420
3421     foreach_drive($conf, sub {
3422         my ($ds, $drive) = @_;
3423
3424         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3425             push @$vollist, $drive->{file};
3426         }
3427
3428         # ignore efidisk here, already added in bios/fw handling code above
3429         return if $drive->{interface} eq 'efidisk';
3430
3431         $use_virtio = 1 if $ds =~ m/^virtio/;
3432
3433         if (drive_is_cdrom ($drive)) {
3434             if ($bootindex_hash->{d}) {
3435                 $drive->{bootindex} = $bootindex_hash->{d};
3436                 $bootindex_hash->{d} += 1;
3437             }
3438         } else {
3439             if ($bootindex_hash->{c}) {
3440                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3441                 $bootindex_hash->{c} += 1;
3442             }
3443         }
3444
3445         if($drive->{interface} eq 'virtio'){
3446            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3447         }
3448
3449         if ($drive->{interface} eq 'scsi') {
3450
3451             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3452
3453             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3454                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3455
3456             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3457             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3458
3459             my $iothread = '';
3460             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3461                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3462                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3463             } elsif ($drive->{iothread}) {
3464                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3465             }
3466
3467             my $queues = '';
3468             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3469                 $queues = ",num_queues=$drive->{queues}";
3470             }
3471
3472             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3473             $scsicontroller->{$controller}=1;
3474         }
3475
3476         if ($drive->{interface} eq 'sata') {
3477            my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3478            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3479            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3480            $ahcicontroller->{$controller}=1;
3481         }
3482
3483         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3484         push @$devices, '-drive',$drive_cmd;
3485         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3486     });
3487
3488     for (my $i = 0; $i < $MAX_NETS; $i++) {
3489          next if !$conf->{"net$i"};
3490          my $d = parse_net($conf->{"net$i"});
3491          next if !$d;
3492
3493          $use_virtio = 1 if $d->{model} eq 'virtio';
3494
3495          if ($bootindex_hash->{n}) {
3496             $d->{bootindex} = $bootindex_hash->{n};
3497             $bootindex_hash->{n} += 1;
3498          }
3499
3500          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3501          push @$devices, '-netdev', $netdevfull;
3502
3503          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3504          push @$devices, '-device', $netdevicefull;
3505     }
3506
3507     if ($conf->{ivshmem}) {
3508         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3509
3510         my $bus;
3511         if ($q35) {
3512             $bus = print_pcie_addr("ivshmem");
3513         } else {
3514             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3515         }
3516
3517         my $ivshmem_name = $ivshmem->{name} // $vmid;
3518         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3519
3520         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3521         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3522     }
3523
3524     # pci.4 is nested in pci.1
3525     $bridges->{1} = 1 if $bridges->{4};
3526
3527     if (!$q35) {
3528         # add pci bridges
3529         if (min_version($machine_version, 2, 3)) {
3530            $bridges->{1} = 1;
3531            $bridges->{2} = 1;
3532         }
3533
3534         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3535
3536     }
3537
3538     for my $k (sort {$b cmp $a} keys %$bridges) {
3539         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3540         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3541         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3542         if ($q35) {
3543             # add after -readconfig pve-q35.cfg
3544             splice @$devices, 2, 0, '-device', $devstr;
3545         } else {
3546             unshift @$devices, '-device', $devstr if $k > 0;
3547         }
3548     }
3549
3550     if (!$kvm) {
3551         push @$machineFlags, 'accel=tcg';
3552     }
3553
3554     my $machine_type_min = $machine_type;
3555     if ($add_pve_version) {
3556         $machine_type_min =~ s/\+pve\d+$//;
3557         $machine_type_min .= "+pve$required_pve_version";
3558     }
3559     push @$machineFlags, "type=${machine_type_min}";
3560
3561     push @$cmd, @$devices;
3562     push @$cmd, '-rtc', join(',', @$rtcFlags)
3563         if scalar(@$rtcFlags);
3564     push @$cmd, '-machine', join(',', @$machineFlags)
3565         if scalar(@$machineFlags);
3566     push @$cmd, '-global', join(',', @$globalFlags)
3567         if scalar(@$globalFlags);
3568
3569     if (my $vmstate = $conf->{vmstate}) {
3570         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3571         push @$vollist, $vmstate;
3572         push @$cmd, '-loadstate', $statepath;
3573         print "activating and using '$vmstate' as vmstate\n";
3574     }
3575
3576     # add custom args
3577     if ($conf->{args}) {
3578         my $aa = PVE::Tools::split_args($conf->{args});
3579         push @$cmd, @$aa;
3580     }
3581
3582     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3583 }
3584
3585 sub spice_port {
3586     my ($vmid) = @_;
3587
3588     my $res = mon_cmd($vmid, 'query-spice');
3589
3590     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3591 }
3592
3593 sub vm_devices_list {
3594     my ($vmid) = @_;
3595
3596     my $res = mon_cmd($vmid, 'query-pci');
3597     my $devices_to_check = [];
3598     my $devices = {};
3599     foreach my $pcibus (@$res) {
3600         push @$devices_to_check, @{$pcibus->{devices}},
3601     }
3602
3603     while (@$devices_to_check) {
3604         my $to_check = [];
3605         for my $d (@$devices_to_check) {
3606             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3607             next if !$d->{'pci_bridge'};
3608
3609             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3610             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3611         }
3612         $devices_to_check = $to_check;
3613     }
3614
3615     my $resblock = mon_cmd($vmid, 'query-block');
3616     foreach my $block (@$resblock) {
3617         if($block->{device} =~ m/^drive-(\S+)/){
3618                 $devices->{$1} = 1;
3619         }
3620     }
3621
3622     my $resmice = mon_cmd($vmid, 'query-mice');
3623     foreach my $mice (@$resmice) {
3624         if ($mice->{name} eq 'QEMU HID Tablet') {
3625             $devices->{tablet} = 1;
3626             last;
3627         }
3628     }
3629
3630     # for usb devices there is no query-usb
3631     # but we can iterate over the entries in
3632     # qom-list path=/machine/peripheral
3633     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3634     foreach my $per (@$resperipheral) {
3635         if ($per->{name} =~ m/^usb\d+$/) {
3636             $devices->{$per->{name}} = 1;
3637         }
3638     }
3639
3640     return $devices;
3641 }
3642
3643 sub vm_deviceplug {
3644     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3645
3646     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3647
3648     my $devices_list = vm_devices_list($vmid);
3649     return 1 if defined($devices_list->{$deviceid});
3650
3651     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3652
3653     if ($deviceid eq 'tablet') {
3654
3655         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3656
3657     } elsif ($deviceid eq 'keyboard') {
3658
3659         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3660
3661     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3662
3663         die "usb hotplug currently not reliable\n";
3664         # since we can't reliably hot unplug all added usb devices
3665         # and usb passthrough disables live migration
3666         # we disable usb hotplugging for now
3667         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3668
3669     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3670
3671         qemu_iothread_add($vmid, $deviceid, $device);
3672
3673         qemu_driveadd($storecfg, $vmid, $device);
3674         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3675
3676         qemu_deviceadd($vmid, $devicefull);
3677         eval { qemu_deviceaddverify($vmid, $deviceid); };
3678         if (my $err = $@) {
3679             eval { qemu_drivedel($vmid, $deviceid); };
3680             warn $@ if $@;
3681             die $err;
3682         }
3683
3684     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3685
3686
3687         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3688         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3689         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3690
3691         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3692
3693         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3694             qemu_iothread_add($vmid, $deviceid, $device);
3695             $devicefull .= ",iothread=iothread-$deviceid";
3696         }
3697
3698         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3699             $devicefull .= ",num_queues=$device->{queues}";
3700         }
3701
3702         qemu_deviceadd($vmid, $devicefull);
3703         qemu_deviceaddverify($vmid, $deviceid);
3704
3705     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3706
3707         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3708         qemu_driveadd($storecfg, $vmid, $device);
3709
3710         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3711         eval { qemu_deviceadd($vmid, $devicefull); };
3712         if (my $err = $@) {
3713             eval { qemu_drivedel($vmid, $deviceid); };
3714             warn $@ if $@;
3715             die $err;
3716         }
3717
3718     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3719
3720         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3721
3722         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3723         my $use_old_bios_files = undef;
3724         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3725
3726         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3727         qemu_deviceadd($vmid, $netdevicefull);
3728         eval {
3729             qemu_deviceaddverify($vmid, $deviceid);
3730             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3731         };
3732         if (my $err = $@) {
3733             eval { qemu_netdevdel($vmid, $deviceid); };
3734             warn $@ if $@;
3735             die $err;
3736         }
3737
3738     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3739
3740         my $bridgeid = $2;
3741         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3742         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3743
3744         qemu_deviceadd($vmid, $devicefull);
3745         qemu_deviceaddverify($vmid, $deviceid);
3746
3747     } else {
3748         die "can't hotplug device '$deviceid'\n";
3749     }
3750
3751     return 1;
3752 }
3753
3754 # fixme: this should raise exceptions on error!
3755 sub vm_deviceunplug {
3756     my ($vmid, $conf, $deviceid) = @_;
3757
3758     my $devices_list = vm_devices_list($vmid);
3759     return 1 if !defined($devices_list->{$deviceid});
3760
3761     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3762
3763     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3764
3765         qemu_devicedel($vmid, $deviceid);
3766
3767     } elsif ($deviceid =~ m/^usb\d+$/) {
3768
3769         die "usb hotplug currently not reliable\n";
3770         # when unplugging usb devices this way,
3771         # there may be remaining usb controllers/hubs
3772         # so we disable it for now
3773         qemu_devicedel($vmid, $deviceid);
3774         qemu_devicedelverify($vmid, $deviceid);
3775
3776     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3777
3778         qemu_devicedel($vmid, $deviceid);
3779         qemu_devicedelverify($vmid, $deviceid);
3780         qemu_drivedel($vmid, $deviceid);
3781         qemu_iothread_del($conf, $vmid, $deviceid);
3782
3783     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3784
3785         qemu_devicedel($vmid, $deviceid);
3786         qemu_devicedelverify($vmid, $deviceid);
3787         qemu_iothread_del($conf, $vmid, $deviceid);
3788
3789     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3790
3791         qemu_devicedel($vmid, $deviceid);
3792         qemu_drivedel($vmid, $deviceid);
3793         qemu_deletescsihw($conf, $vmid, $deviceid);
3794
3795     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3796
3797         qemu_devicedel($vmid, $deviceid);
3798         qemu_devicedelverify($vmid, $deviceid);
3799         qemu_netdevdel($vmid, $deviceid);
3800
3801     } else {
3802         die "can't unplug device '$deviceid'\n";
3803     }
3804
3805     return 1;
3806 }
3807
3808 sub qemu_deviceadd {
3809     my ($vmid, $devicefull) = @_;
3810
3811     $devicefull = "driver=".$devicefull;
3812     my %options =  split(/[=,]/, $devicefull);
3813
3814     mon_cmd($vmid, "device_add" , %options);
3815 }
3816
3817 sub qemu_devicedel {
3818     my ($vmid, $deviceid) = @_;
3819
3820     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3821 }
3822
3823 sub qemu_iothread_add {
3824     my($vmid, $deviceid, $device) = @_;
3825
3826     if ($device->{iothread}) {
3827         my $iothreads = vm_iothreads_list($vmid);
3828         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3829     }
3830 }
3831
3832 sub qemu_iothread_del {
3833     my($conf, $vmid, $deviceid) = @_;
3834
3835     my $confid = $deviceid;
3836     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3837         $confid = 'scsi' . $1;
3838     }
3839     my $device = parse_drive($confid, $conf->{$confid});
3840     if ($device->{iothread}) {
3841         my $iothreads = vm_iothreads_list($vmid);
3842         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3843     }
3844 }
3845
3846 sub qemu_objectadd {
3847     my($vmid, $objectid, $qomtype) = @_;
3848
3849     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3850
3851     return 1;
3852 }
3853
3854 sub qemu_objectdel {
3855     my($vmid, $objectid) = @_;
3856
3857     mon_cmd($vmid, "object-del", id => $objectid);
3858
3859     return 1;
3860 }
3861
3862 sub qemu_driveadd {
3863     my ($storecfg, $vmid, $device) = @_;
3864
3865     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3866     $drive =~ s/\\/\\\\/g;
3867     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3868
3869     # If the command succeeds qemu prints: "OK"
3870     return 1 if $ret =~ m/OK/s;
3871
3872     die "adding drive failed: $ret\n";
3873 }
3874
3875 sub qemu_drivedel {
3876     my($vmid, $deviceid) = @_;
3877
3878     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3879     $ret =~ s/^\s+//;
3880
3881     return 1 if $ret eq "";
3882
3883     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3884     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3885
3886     die "deleting drive $deviceid failed : $ret\n";
3887 }
3888
3889 sub qemu_deviceaddverify {
3890     my ($vmid, $deviceid) = @_;
3891
3892     for (my $i = 0; $i <= 5; $i++) {
3893          my $devices_list = vm_devices_list($vmid);
3894          return 1 if defined($devices_list->{$deviceid});
3895          sleep 1;
3896     }
3897
3898     die "error on hotplug device '$deviceid'\n";
3899 }
3900
3901
3902 sub qemu_devicedelverify {
3903     my ($vmid, $deviceid) = @_;
3904
3905     # need to verify that the device is correctly removed as device_del
3906     # is async and empty return is not reliable
3907
3908     for (my $i = 0; $i <= 5; $i++) {
3909          my $devices_list = vm_devices_list($vmid);
3910          return 1 if !defined($devices_list->{$deviceid});
3911          sleep 1;
3912     }
3913
3914     die "error on hot-unplugging device '$deviceid'\n";
3915 }
3916
3917 sub qemu_findorcreatescsihw {
3918     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3919
3920     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3921
3922     my $scsihwid="$controller_prefix$controller";
3923     my $devices_list = vm_devices_list($vmid);
3924
3925     if(!defined($devices_list->{$scsihwid})) {
3926         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3927     }
3928
3929     return 1;
3930 }
3931
3932 sub qemu_deletescsihw {
3933     my ($conf, $vmid, $opt) = @_;
3934
3935     my $device = parse_drive($opt, $conf->{$opt});
3936
3937     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3938         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3939         return 1;
3940     }
3941
3942     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3943
3944     my $devices_list = vm_devices_list($vmid);
3945     foreach my $opt (keys %{$devices_list}) {
3946         if (is_valid_drivename($opt)) {
3947             my $drive = parse_drive($opt, $conf->{$opt});
3948             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3949                 return 1;
3950             }
3951         }
3952     }
3953
3954     my $scsihwid="scsihw$controller";
3955
3956     vm_deviceunplug($vmid, $conf, $scsihwid);
3957
3958     return 1;
3959 }
3960
3961 sub qemu_add_pci_bridge {
3962     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3963
3964     my $bridges = {};
3965
3966     my $bridgeid;
3967
3968     print_pci_addr($device, $bridges, $arch, $machine_type);
3969
3970     while (my ($k, $v) = each %$bridges) {
3971         $bridgeid = $k;
3972     }
3973     return 1 if !defined($bridgeid) || $bridgeid < 1;
3974
3975     my $bridge = "pci.$bridgeid";
3976     my $devices_list = vm_devices_list($vmid);
3977
3978     if (!defined($devices_list->{$bridge})) {
3979         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3980     }
3981
3982     return 1;
3983 }
3984
3985 sub qemu_set_link_status {
3986     my ($vmid, $device, $up) = @_;
3987
3988     mon_cmd($vmid, "set_link", name => $device,
3989                up => $up ? JSON::true : JSON::false);
3990 }
3991
3992 sub qemu_netdevadd {
3993     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3994
3995     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3996     my %options =  split(/[=,]/, $netdev);
3997
3998     mon_cmd($vmid, "netdev_add",  %options);
3999     return 1;
4000 }
4001
4002 sub qemu_netdevdel {
4003     my ($vmid, $deviceid) = @_;
4004
4005     mon_cmd($vmid, "netdev_del", id => $deviceid);
4006 }
4007
4008 sub qemu_usb_hotplug {
4009     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4010
4011     return if !$device;
4012
4013     # remove the old one first
4014     vm_deviceunplug($vmid, $conf, $deviceid);
4015
4016     # check if xhci controller is necessary and available
4017     if ($device->{usb3}) {
4018
4019         my $devicelist = vm_devices_list($vmid);
4020
4021         if (!$devicelist->{xhci}) {
4022             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4023             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4024         }
4025     }
4026     my $d = parse_usb_device($device->{host});
4027     $d->{usb3} = $device->{usb3};
4028
4029     # add the new one
4030     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4031 }
4032
4033 sub qemu_cpu_hotplug {
4034     my ($vmid, $conf, $vcpus) = @_;
4035
4036     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4037
4038     my $sockets = 1;
4039     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4040     $sockets = $conf->{sockets} if  $conf->{sockets};
4041     my $cores = $conf->{cores} || 1;
4042     my $maxcpus = $sockets * $cores;
4043
4044     $vcpus = $maxcpus if !$vcpus;
4045
4046     die "you can't add more vcpus than maxcpus\n"
4047         if $vcpus > $maxcpus;
4048
4049     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4050
4051     if ($vcpus < $currentvcpus) {
4052
4053         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4054
4055             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4056                 qemu_devicedel($vmid, "cpu$i");
4057                 my $retry = 0;
4058                 my $currentrunningvcpus = undef;
4059                 while (1) {
4060                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4061                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4062                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4063                     $retry++;
4064                     sleep 1;
4065                 }
4066                 #update conf after each succesfull cpu unplug
4067                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4068                 PVE::QemuConfig->write_config($vmid, $conf);
4069             }
4070         } else {
4071             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4072         }
4073
4074         return;
4075     }
4076
4077     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4078     die "vcpus in running vm does not match its configuration\n"
4079         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4080
4081     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4082
4083         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4084             my $cpustr = print_cpu_device($conf, $i);
4085             qemu_deviceadd($vmid, $cpustr);
4086
4087             my $retry = 0;
4088             my $currentrunningvcpus = undef;
4089             while (1) {
4090                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4091                 last if scalar(@{$currentrunningvcpus}) == $i;
4092                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4093                 sleep 1;
4094                 $retry++;
4095             }
4096             #update conf after each succesfull cpu hotplug
4097             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4098             PVE::QemuConfig->write_config($vmid, $conf);
4099         }
4100     } else {
4101
4102         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4103             mon_cmd($vmid, "cpu-add", id => int($i));
4104         }
4105     }
4106 }
4107
4108 sub qemu_block_set_io_throttle {
4109     my ($vmid, $deviceid,
4110         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4111         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4112         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4113         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4114
4115     return if !check_running($vmid) ;
4116
4117     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4118         bps => int($bps),
4119         bps_rd => int($bps_rd),
4120         bps_wr => int($bps_wr),
4121         iops => int($iops),
4122         iops_rd => int($iops_rd),
4123         iops_wr => int($iops_wr),
4124         bps_max => int($bps_max),
4125         bps_rd_max => int($bps_rd_max),
4126         bps_wr_max => int($bps_wr_max),
4127         iops_max => int($iops_max),
4128         iops_rd_max => int($iops_rd_max),
4129         iops_wr_max => int($iops_wr_max),
4130         bps_max_length => int($bps_max_length),
4131         bps_rd_max_length => int($bps_rd_max_length),
4132         bps_wr_max_length => int($bps_wr_max_length),
4133         iops_max_length => int($iops_max_length),
4134         iops_rd_max_length => int($iops_rd_max_length),
4135         iops_wr_max_length => int($iops_wr_max_length),
4136     );
4137
4138 }
4139
4140 # old code, only used to shutdown old VM after update
4141 sub __read_avail {
4142     my ($fh, $timeout) = @_;
4143
4144     my $sel = new IO::Select;
4145     $sel->add($fh);
4146
4147     my $res = '';
4148     my $buf;
4149
4150     my @ready;
4151     while (scalar (@ready = $sel->can_read($timeout))) {
4152         my $count;
4153         if ($count = $fh->sysread($buf, 8192)) {
4154             if ($buf =~ /^(.*)\(qemu\) $/s) {
4155                 $res .= $1;
4156                 last;
4157             } else {
4158                 $res .= $buf;
4159             }
4160         } else {
4161             if (!defined($count)) {
4162                 die "$!\n";
4163             }
4164             last;
4165         }
4166     }
4167
4168     die "monitor read timeout\n" if !scalar(@ready);
4169
4170     return $res;
4171 }
4172
4173 sub qemu_block_resize {
4174     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4175
4176     my $running = check_running($vmid);
4177
4178     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4179
4180     return if !$running;
4181
4182     my $padding = (1024 - $size % 1024) % 1024;
4183     $size = $size + $padding;
4184
4185     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4186
4187 }
4188
4189 sub qemu_volume_snapshot {
4190     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4191
4192     my $running = check_running($vmid);
4193
4194     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4195         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4196     } else {
4197         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4198     }
4199 }
4200
4201 sub qemu_volume_snapshot_delete {
4202     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4203
4204     my $running = check_running($vmid);
4205
4206     if($running) {
4207
4208         $running = undef;
4209         my $conf = PVE::QemuConfig->load_config($vmid);
4210         foreach_drive($conf, sub {
4211             my ($ds, $drive) = @_;
4212             $running = 1 if $drive->{file} eq $volid;
4213         });
4214     }
4215
4216     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4217         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4218     } else {
4219         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4220     }
4221 }
4222
4223 sub set_migration_caps {
4224     my ($vmid) = @_;
4225
4226     my $cap_ref = [];
4227
4228     my $enabled_cap = {
4229         "auto-converge" => 1,
4230         "xbzrle" => 1,
4231         "x-rdma-pin-all" => 0,
4232         "zero-blocks" => 0,
4233         "compress" => 0
4234     };
4235
4236     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4237
4238     for my $supported_capability (@$supported_capabilities) {
4239         push @$cap_ref, {
4240             capability => $supported_capability->{capability},
4241             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4242         };
4243     }
4244
4245     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4246 }
4247
4248 my $fast_plug_option = {
4249     'lock' => 1,
4250     'name' => 1,
4251     'onboot' => 1,
4252     'shares' => 1,
4253     'startup' => 1,
4254     'description' => 1,
4255     'protection' => 1,
4256     'vmstatestorage' => 1,
4257     'hookscript' => 1,
4258     'tags' => 1,
4259 };
4260
4261 # hotplug changes in [PENDING]
4262 # $selection hash can be used to only apply specified options, for
4263 # example: { cores => 1 } (only apply changed 'cores')
4264 # $errors ref is used to return error messages
4265 sub vmconfig_hotplug_pending {
4266     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4267
4268     my $defaults = load_defaults();
4269     my $arch = get_vm_arch($conf);
4270     my $machine_type = get_vm_machine($conf, undef, $arch);
4271
4272     # commit values which do not have any impact on running VM first
4273     # Note: those option cannot raise errors, we we do not care about
4274     # $selection and always apply them.
4275
4276     my $add_error = sub {
4277         my ($opt, $msg) = @_;
4278         $errors->{$opt} = "hotplug problem - $msg";
4279     };
4280
4281     my $changes = 0;
4282     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4283         if ($fast_plug_option->{$opt}) {
4284             $conf->{$opt} = $conf->{pending}->{$opt};
4285             delete $conf->{pending}->{$opt};
4286             $changes = 1;
4287         }
4288     }
4289
4290     if ($changes) {
4291         PVE::QemuConfig->write_config($vmid, $conf);
4292     }
4293
4294     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4295
4296     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4297     foreach my $opt (sort keys %$pending_delete_hash) {
4298         next if $selection && !$selection->{$opt};
4299         my $force = $pending_delete_hash->{$opt}->{force};
4300         eval {
4301             if ($opt eq 'hotplug') {
4302                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4303             } elsif ($opt eq 'tablet') {
4304                 die "skip\n" if !$hotplug_features->{usb};
4305                 if ($defaults->{tablet}) {
4306                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4307                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4308                         if $arch eq 'aarch64';
4309                 } else {
4310                     vm_deviceunplug($vmid, $conf, 'tablet');
4311                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4312                 }
4313             } elsif ($opt =~ m/^usb\d+/) {
4314                 die "skip\n";
4315                 # since we cannot reliably hot unplug usb devices
4316                 # we are disabling it
4317                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4318                 vm_deviceunplug($vmid, $conf, $opt);
4319             } elsif ($opt eq 'vcpus') {
4320                 die "skip\n" if !$hotplug_features->{cpu};
4321                 qemu_cpu_hotplug($vmid, $conf, undef);
4322             } elsif ($opt eq 'balloon') {
4323                 # enable balloon device is not hotpluggable
4324                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4325                 # here we reset the ballooning value to memory
4326                 my $balloon = $conf->{memory} || $defaults->{memory};
4327                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4328             } elsif ($fast_plug_option->{$opt}) {
4329                 # do nothing
4330             } elsif ($opt =~ m/^net(\d+)$/) {
4331                 die "skip\n" if !$hotplug_features->{network};
4332                 vm_deviceunplug($vmid, $conf, $opt);
4333             } elsif (is_valid_drivename($opt)) {
4334                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4335                 vm_deviceunplug($vmid, $conf, $opt);
4336                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4337             } elsif ($opt =~ m/^memory$/) {
4338                 die "skip\n" if !$hotplug_features->{memory};
4339                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4340             } elsif ($opt eq 'cpuunits') {
4341                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4342             } elsif ($opt eq 'cpulimit') {
4343                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4344             } else {
4345                 die "skip\n";
4346             }
4347         };
4348         if (my $err = $@) {
4349             &$add_error($opt, $err) if $err ne "skip\n";
4350         } else {
4351             delete $conf->{$opt};
4352             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4353         }
4354     }
4355
4356     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4357     $apply_pending_cloudinit = sub {
4358         return if $apply_pending_cloudinit_done; # once is enough
4359         $apply_pending_cloudinit_done = 1; # once is enough
4360
4361         my ($key, $value) = @_;
4362
4363         my @cloudinit_opts = keys %$confdesc_cloudinit;
4364         foreach my $opt (keys %{$conf->{pending}}) {
4365             next if !grep { $_ eq $opt } @cloudinit_opts;
4366             $conf->{$opt} = delete $conf->{pending}->{$opt};
4367         }
4368
4369         my $new_conf = { %$conf };
4370         $new_conf->{$key} = $value;
4371         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4372     };
4373
4374     foreach my $opt (keys %{$conf->{pending}}) {
4375         next if $selection && !$selection->{$opt};
4376         my $value = $conf->{pending}->{$opt};
4377         eval {
4378             if ($opt eq 'hotplug') {
4379                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4380             } elsif ($opt eq 'tablet') {
4381                 die "skip\n" if !$hotplug_features->{usb};
4382                 if ($value == 1) {
4383                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4384                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4385                         if $arch eq 'aarch64';
4386                 } elsif ($value == 0) {
4387                     vm_deviceunplug($vmid, $conf, 'tablet');
4388                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4389                 }
4390             } elsif ($opt =~ m/^usb\d+$/) {
4391                 die "skip\n";
4392                 # since we cannot reliably hot unplug usb devices
4393                 # we are disabling it
4394                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4395                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4396                 die "skip\n" if !$d;
4397                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4398             } elsif ($opt eq 'vcpus') {
4399                 die "skip\n" if !$hotplug_features->{cpu};
4400                 qemu_cpu_hotplug($vmid, $conf, $value);
4401             } elsif ($opt eq 'balloon') {
4402                 # enable/disable balloning device is not hotpluggable
4403                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4404                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4405                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4406
4407                 # allow manual ballooning if shares is set to zero
4408                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4409                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4410                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4411                 }
4412             } elsif ($opt =~ m/^net(\d+)$/) {
4413                 # some changes can be done without hotplug
4414                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4415                                     $vmid, $opt, $value, $arch, $machine_type);
4416             } elsif (is_valid_drivename($opt)) {
4417                 die "skip\n" if $opt eq 'efidisk0';
4418                 # some changes can be done without hotplug
4419                 my $drive = parse_drive($opt, $value);
4420                 if (drive_is_cloudinit($drive)) {
4421                     &$apply_pending_cloudinit($opt, $value);
4422                 }
4423                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4424                                      $vmid, $opt, $value, $arch, $machine_type);
4425             } elsif ($opt =~ m/^memory$/) { #dimms
4426                 die "skip\n" if !$hotplug_features->{memory};
4427                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4428             } elsif ($opt eq 'cpuunits') {
4429                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4430             } elsif ($opt eq 'cpulimit') {
4431                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4432                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4433             } else {
4434                 die "skip\n";  # skip non-hot-pluggable options
4435             }
4436         };
4437         if (my $err = $@) {
4438             &$add_error($opt, $err) if $err ne "skip\n";
4439         } else {
4440             $conf->{$opt} = $value;
4441             delete $conf->{pending}->{$opt};
4442         }
4443     }
4444
4445     PVE::QemuConfig->write_config($vmid, $conf);
4446 }
4447
4448 sub try_deallocate_drive {
4449     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4450
4451     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4452         my $volid = $drive->{file};
4453         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4454             my $sid = PVE::Storage::parse_volume_id($volid);
4455             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4456
4457             # check if the disk is really unused
4458             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4459                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4460             PVE::Storage::vdisk_free($storecfg, $volid);
4461             return 1;
4462         } else {
4463             # If vm is not owner of this disk remove from config
4464             return 1;
4465         }
4466     }
4467
4468     return undef;
4469 }
4470
4471 sub vmconfig_delete_or_detach_drive {
4472     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4473
4474     my $drive = parse_drive($opt, $conf->{$opt});
4475
4476     my $rpcenv = PVE::RPCEnvironment::get();
4477     my $authuser = $rpcenv->get_user();
4478
4479     if ($force) {
4480         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4481         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4482     } else {
4483         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4484     }
4485 }
4486
4487
4488
4489 sub vmconfig_apply_pending {
4490     my ($vmid, $conf, $storecfg, $errors) = @_;
4491
4492     my $add_apply_error = sub {
4493         my ($opt, $msg) = @_;
4494         my $err_msg = "unable to apply pending change $opt : $msg";
4495         $errors->{$opt} = $err_msg;
4496         warn $err_msg;
4497     };
4498
4499     # cold plug
4500
4501     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4502     foreach my $opt (sort keys %$pending_delete_hash) {
4503         my $force = $pending_delete_hash->{$opt}->{force};
4504         eval {
4505             if ($opt =~ m/^unused/) {
4506                 die "internal error";
4507             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4508                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4509             }
4510         };
4511         if (my $err = $@) {
4512             $add_apply_error->($opt, $err);
4513         } else {
4514             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4515             delete $conf->{$opt};
4516         }
4517     }
4518
4519     PVE::QemuConfig->cleanup_pending($conf);
4520
4521     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4522         next if $opt eq 'delete'; # just to be sure
4523         eval {
4524             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4525                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4526             }
4527         };
4528         if (my $err = $@) {
4529             $add_apply_error->($opt, $err);
4530         } else {
4531             $conf->{$opt} = delete $conf->{pending}->{$opt};
4532         }
4533     }
4534
4535     # write all changes at once to avoid unnecessary i/o
4536     PVE::QemuConfig->write_config($vmid, $conf);
4537 }
4538
4539 my $safe_num_ne = sub {
4540     my ($a, $b) = @_;
4541
4542     return 0 if !defined($a) && !defined($b);
4543     return 1 if !defined($a);
4544     return 1 if !defined($b);
4545
4546     return $a != $b;
4547 };
4548
4549 my $safe_string_ne = sub {
4550     my ($a, $b) = @_;
4551
4552     return 0 if !defined($a) && !defined($b);
4553     return 1 if !defined($a);
4554     return 1 if !defined($b);
4555
4556     return $a ne $b;
4557 };
4558
4559 sub vmconfig_update_net {
4560     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4561
4562     my $newnet = parse_net($value);
4563
4564     if ($conf->{$opt}) {
4565         my $oldnet = parse_net($conf->{$opt});
4566
4567         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4568             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4569             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4570             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4571
4572             # for non online change, we try to hot-unplug
4573             die "skip\n" if !$hotplug;
4574             vm_deviceunplug($vmid, $conf, $opt);
4575         } else {
4576
4577             die "internal error" if $opt !~ m/net(\d+)/;
4578             my $iface = "tap${vmid}i$1";
4579
4580             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4581                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4582                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4583                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4584                 PVE::Network::tap_unplug($iface);
4585                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4586             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4587                 # Rate can be applied on its own but any change above needs to
4588                 # include the rate in tap_plug since OVS resets everything.
4589                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4590             }
4591
4592             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4593                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4594             }
4595
4596             return 1;
4597         }
4598     }
4599
4600     if ($hotplug) {
4601         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4602     } else {
4603         die "skip\n";
4604     }
4605 }
4606
4607 sub vmconfig_update_disk {
4608     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4609
4610     my $drive = parse_drive($opt, $value);
4611
4612     if ($conf->{$opt}) {
4613
4614         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4615
4616             my $media = $drive->{media} || 'disk';
4617             my $oldmedia = $old_drive->{media} || 'disk';
4618             die "unable to change media type\n" if $media ne $oldmedia;
4619
4620             if (!drive_is_cdrom($old_drive)) {
4621
4622                 if ($drive->{file} ne $old_drive->{file}) {
4623
4624                     die "skip\n" if !$hotplug;
4625
4626                     # unplug and register as unused
4627                     vm_deviceunplug($vmid, $conf, $opt);
4628                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4629
4630                 } else {
4631                     # update existing disk
4632
4633                     # skip non hotpluggable value
4634                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4635                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4636                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4637                         &$safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4638                         &$safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4639                         die "skip\n";
4640                     }
4641
4642                     # apply throttle
4643                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4644                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4645                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4646                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4647                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4648                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4649                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4650                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4651                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4652                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4653                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4654                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4655                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4656                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4657                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4658                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4659                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4660                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4661
4662                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4663                                                    ($drive->{mbps} || 0)*1024*1024,
4664                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4665                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4666                                                    $drive->{iops} || 0,
4667                                                    $drive->{iops_rd} || 0,
4668                                                    $drive->{iops_wr} || 0,
4669                                                    ($drive->{mbps_max} || 0)*1024*1024,
4670                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4671                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4672                                                    $drive->{iops_max} || 0,
4673                                                    $drive->{iops_rd_max} || 0,
4674                                                    $drive->{iops_wr_max} || 0,
4675                                                    $drive->{bps_max_length} || 1,
4676                                                    $drive->{bps_rd_max_length} || 1,
4677                                                    $drive->{bps_wr_max_length} || 1,
4678                                                    $drive->{iops_max_length} || 1,
4679                                                    $drive->{iops_rd_max_length} || 1,
4680                                                    $drive->{iops_wr_max_length} || 1);
4681
4682                     }
4683
4684                     return 1;
4685                 }
4686
4687             } else { # cdrom
4688
4689                 if ($drive->{file} eq 'none') {
4690                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4691                     if (drive_is_cloudinit($old_drive)) {
4692                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4693                     }
4694                 } else {
4695                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4696
4697                     # force eject if locked
4698                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4699
4700                     if ($path) {
4701                         mon_cmd($vmid, "blockdev-change-medium",
4702                             id => "$opt", filename => "$path");
4703                     }
4704                 }
4705
4706                 return 1;
4707             }
4708         }
4709     }
4710
4711     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4712     # hotplug new disks
4713     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4714     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4715 }
4716
4717 sub vm_start {
4718     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4719         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
4720
4721     PVE::QemuConfig->lock_config($vmid, sub {
4722         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4723
4724         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4725
4726         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
4727
4728         PVE::QemuConfig->check_lock($conf)
4729             if !($skiplock || $is_suspended);
4730
4731         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4732
4733         # clean up leftover reboot request files
4734         eval { clear_reboot_request($vmid); };
4735         warn $@ if $@;
4736
4737         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4738             vmconfig_apply_pending($vmid, $conf, $storecfg);
4739             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4740         }
4741
4742         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4743
4744         my $defaults = load_defaults();
4745
4746         # set environment variable useful inside network script
4747         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4748
4749         my $local_volumes = {};
4750
4751         if ($targetstorage) {
4752             foreach_drive($conf, sub {
4753                 my ($ds, $drive) = @_;
4754
4755                 return if drive_is_cdrom($drive);
4756
4757                 my $volid = $drive->{file};
4758
4759                 return if !$volid;
4760
4761                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4762
4763                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4764                 return if $scfg->{shared};
4765                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4766             });
4767
4768             my $format = undef;
4769
4770             foreach my $opt (sort keys %$local_volumes) {
4771
4772                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4773                 my $drive = parse_drive($opt, $conf->{$opt});
4774
4775                 # If a remote storage is specified and the format of the original
4776                 # volume is not available there, fall back to the default format.
4777                 # Otherwise use the same format as the original.
4778                 if ($targetstorage && $targetstorage ne "1") {
4779                     $storeid = $targetstorage;
4780                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4781                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4782                     my $fileFormat = qemu_img_format($scfg, $volname);
4783                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4784                 } else {
4785                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4786                     $format = qemu_img_format($scfg, $volname);
4787                 }
4788
4789                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4790                 my $newdrive = $drive;
4791                 $newdrive->{format} = $format;
4792                 $newdrive->{file} = $newvolid;
4793                 my $drivestr = print_drive($newdrive);
4794                 $local_volumes->{$opt} = $drivestr;
4795                 #pass drive to conf for command line
4796                 $conf->{$opt} = $drivestr;
4797             }
4798         }
4799
4800         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4801
4802         if ($is_suspended) {
4803             # enforce machine type on suspended vm to ensure HW compatibility
4804             $forcemachine = $conf->{runningmachine};
4805             print "Resuming suspended VM\n";
4806         }
4807
4808         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4809
4810         my $migration_ip;
4811         my $get_migration_ip = sub {
4812             my ($cidr, $nodename) = @_;
4813
4814             return $migration_ip if defined($migration_ip);
4815
4816             if (!defined($cidr)) {
4817                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4818                 $cidr = $dc_conf->{migration}->{network};
4819             }
4820
4821             if (defined($cidr)) {
4822                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4823
4824                 die "could not get IP: no address configured on local " .
4825                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
4826
4827                 die "could not get IP: multiple addresses configured on local " .
4828                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
4829
4830                 $migration_ip = @$ips[0];
4831             }
4832
4833             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4834                 if !defined($migration_ip);
4835
4836             return $migration_ip;
4837         };
4838
4839         my $migrate_uri;
4840         if ($statefile) {
4841             if ($statefile eq 'tcp') {
4842                 my $localip = "localhost";
4843                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4844                 my $nodename = nodename();
4845
4846                 if (!defined($migration_type)) {
4847                     if (defined($datacenterconf->{migration}->{type})) {
4848                         $migration_type = $datacenterconf->{migration}->{type};
4849                     } else {
4850                         $migration_type = 'secure';
4851                     }
4852                 }
4853
4854                 if ($migration_type eq 'insecure') {
4855                     $localip = $get_migration_ip->($migration_network, $nodename);
4856                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4857                 }
4858
4859                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4860                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4861                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4862                 push @$cmd, '-incoming', $migrate_uri;
4863                 push @$cmd, '-S';
4864
4865             } elsif ($statefile eq 'unix') {
4866                 # should be default for secure migrations as a ssh TCP forward
4867                 # tunnel is not deterministic reliable ready and fails regurarly
4868                 # to set up in time, so use UNIX socket forwards
4869                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4870                 unlink $socket_addr;
4871
4872                 $migrate_uri = "unix:$socket_addr";
4873
4874                 push @$cmd, '-incoming', $migrate_uri;
4875                 push @$cmd, '-S';
4876
4877             } elsif (-e $statefile) {
4878                 push @$cmd, '-loadstate', $statefile;
4879             } else {
4880                 my $statepath = PVE::Storage::path($storecfg, $statefile);
4881                 push @$vollist, $statefile;
4882                 push @$cmd, '-loadstate', $statepath;
4883             }
4884         } elsif ($paused) {
4885             push @$cmd, '-S';
4886         }
4887
4888         # host pci devices
4889         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4890           my $d = parse_hostpci($conf->{"hostpci$i"});
4891           next if !$d;
4892           my $pcidevices = $d->{pciid};
4893           foreach my $pcidevice (@$pcidevices) {
4894                 my $pciid = $pcidevice->{id};
4895
4896                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4897                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4898                 die "no pci device info for device '$pciid'\n" if !$info;
4899
4900                 if ($d->{mdev}) {
4901                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4902                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4903                 } else {
4904                     die "can't unbind/bind pci group to vfio '$pciid'\n"
4905                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4906                     die "can't reset pci device '$pciid'\n"
4907                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4908                 }
4909           }
4910         }
4911
4912         PVE::Storage::activate_volumes($storecfg, $vollist);
4913
4914         eval {
4915             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4916                 outfunc => sub {}, errfunc => sub {});
4917         };
4918         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4919         # timeout should be more than enough here...
4920         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4921
4922         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4923                                                   : $defaults->{cpuunits};
4924
4925         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
4926         my %run_params = (
4927             timeout => $statefile ? undef : $start_timeout,
4928             umask => 0077,
4929             noerr => 1,
4930         );
4931
4932         # when migrating, prefix QEMU output so other side can pick up any
4933         # errors that might occur and show the user
4934         if ($migratedfrom) {
4935             $run_params{quiet} = 1;
4936             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4937         }
4938
4939         my %properties = (
4940             Slice => 'qemu.slice',
4941             KillMode => 'none',
4942             CPUShares => $cpuunits
4943         );
4944
4945         if (my $cpulimit = $conf->{cpulimit}) {
4946             $properties{CPUQuota} = int($cpulimit * 100);
4947         }
4948         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4949
4950         my $run_qemu = sub {
4951             PVE::Tools::run_fork sub {
4952                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4953
4954                 my $exitcode = run_command($cmd, %run_params);
4955                 die "QEMU exited with code $exitcode\n" if $exitcode;
4956             };
4957         };
4958
4959         if ($conf->{hugepages}) {
4960
4961             my $code = sub {
4962                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4963                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4964
4965                 PVE::QemuServer::Memory::hugepages_mount();
4966                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4967
4968                 eval { $run_qemu->() };
4969                 if (my $err = $@) {
4970                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4971                     die $err;
4972                 }
4973
4974                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4975             };
4976             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4977
4978         } else {
4979             eval { $run_qemu->() };
4980         }
4981
4982         if (my $err = $@) {
4983             # deactivate volumes if start fails
4984             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4985             die "start failed: $err";
4986         }
4987
4988         print "migration listens on $migrate_uri\n" if $migrate_uri;
4989
4990         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4991             eval { mon_cmd($vmid, "cont"); };
4992             warn $@ if $@;
4993         }
4994
4995         #start nbd server for storage migration
4996         if ($targetstorage) {
4997             my $nodename = nodename();
4998             my $localip = $get_migration_ip->($migration_network, $nodename);
4999             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5000             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5001
5002             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5003
5004             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5005
5006             foreach my $opt (sort keys %$local_volumes) {
5007                 my $drivestr = $local_volumes->{$opt};
5008                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5009                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5010                 print "storage migration listens on $migrate_storage_uri volume:$drivestr\n";
5011             }
5012         }
5013
5014         if ($migratedfrom) {
5015             eval {
5016                 set_migration_caps($vmid);
5017             };
5018             warn $@ if $@;
5019
5020             if ($spice_port) {
5021                 print "spice listens on port $spice_port\n";
5022                 if ($spice_ticket) {
5023                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5024                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5025                 }
5026             }
5027
5028         } else {
5029             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5030                 if !$statefile && $conf->{balloon};
5031
5032             foreach my $opt (keys %$conf) {
5033                 next if $opt !~  m/^net\d+$/;
5034                 my $nicconf = parse_net($conf->{$opt});
5035                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5036             }
5037         }
5038
5039         mon_cmd($vmid, 'qom-set',
5040                     path => "machine/peripheral/balloon0",
5041                     property => "guest-stats-polling-interval",
5042                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5043
5044         if ($is_suspended) {
5045             print "Resumed VM, removing state\n";
5046             if (my $vmstate = $conf->{vmstate}) {
5047                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5048                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5049             }
5050             delete $conf->@{qw(lock vmstate runningmachine)};
5051             PVE::QemuConfig->write_config($vmid, $conf);
5052         }
5053
5054         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5055     });
5056 }
5057
5058 sub vm_commandline {
5059     my ($storecfg, $vmid, $snapname) = @_;
5060
5061     my $conf = PVE::QemuConfig->load_config($vmid);
5062     my $forcemachine;
5063
5064     if ($snapname) {
5065         my $snapshot = $conf->{snapshots}->{$snapname};
5066         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5067
5068         # check for a 'runningmachine' in snapshot
5069         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5070
5071         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5072
5073         $conf = $snapshot;
5074     }
5075
5076     my $defaults = load_defaults();
5077
5078     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5079
5080     return PVE::Tools::cmd2string($cmd);
5081 }
5082
5083 sub vm_reset {
5084     my ($vmid, $skiplock) = @_;
5085
5086     PVE::QemuConfig->lock_config($vmid, sub {
5087
5088         my $conf = PVE::QemuConfig->load_config($vmid);
5089
5090         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5091
5092         mon_cmd($vmid, "system_reset");
5093     });
5094 }
5095
5096 sub get_vm_volumes {
5097     my ($conf) = @_;
5098
5099     my $vollist = [];
5100     foreach_volid($conf, sub {
5101         my ($volid, $attr) = @_;
5102
5103         return if $volid =~ m|^/|;
5104
5105         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5106         return if !$sid;
5107
5108         push @$vollist, $volid;
5109     });
5110
5111     return $vollist;
5112 }
5113
5114 sub vm_stop_cleanup {
5115     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5116
5117     eval {
5118
5119         if (!$keepActive) {
5120             my $vollist = get_vm_volumes($conf);
5121             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5122         }
5123
5124         foreach my $ext (qw(mon qmp pid vnc qga)) {
5125             unlink "/var/run/qemu-server/${vmid}.$ext";
5126         }
5127
5128         if ($conf->{ivshmem}) {
5129             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5130             # just delete it for now, VMs which have this already open do not
5131             # are affected, but new VMs will get a separated one. If this
5132             # becomes an issue we either add some sort of ref-counting or just
5133             # add a "don't delete on stop" flag to the ivshmem format.
5134             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5135         }
5136
5137         foreach my $key (keys %$conf) {
5138             next if $key !~ m/^hostpci(\d+)$/;
5139             my $hostpciindex = $1;
5140             my $d = parse_hostpci($conf->{$key});
5141             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5142
5143             foreach my $pci (@{$d->{pciid}}) {
5144                 my $pciid = $pci->{id};
5145                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5146             }
5147         }
5148
5149         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5150     };
5151     warn $@ if $@; # avoid errors - just warn
5152 }
5153
5154 # call only in locked context
5155 sub _do_vm_stop {
5156     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5157
5158     my $pid = check_running($vmid, $nocheck);
5159     return if !$pid;
5160
5161     my $conf;
5162     if (!$nocheck) {
5163         $conf = PVE::QemuConfig->load_config($vmid);
5164         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5165         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5166             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5167             $timeout = $opts->{down} if $opts->{down};
5168         }
5169         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5170     }
5171
5172     eval {
5173         if ($shutdown) {
5174             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5175                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5176             } else {
5177                 mon_cmd($vmid, "system_powerdown");
5178             }
5179         } else {
5180             mon_cmd($vmid, "quit");
5181         }
5182     };
5183     my $err = $@;
5184
5185     if (!$err) {
5186         $timeout = 60 if !defined($timeout);
5187
5188         my $count = 0;
5189         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5190             $count++;
5191             sleep 1;
5192         }
5193
5194         if ($count >= $timeout) {
5195             if ($force) {
5196                 warn "VM still running - terminating now with SIGTERM\n";
5197                 kill 15, $pid;
5198             } else {
5199                 die "VM quit/powerdown failed - got timeout\n";
5200             }
5201         } else {
5202             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5203             return;
5204         }
5205     } else {
5206         if ($force) {
5207             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5208             kill 15, $pid;
5209         } else {
5210             die "VM quit/powerdown failed\n";
5211         }
5212     }
5213
5214     # wait again
5215     $timeout = 10;
5216
5217     my $count = 0;
5218     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5219         $count++;
5220         sleep 1;
5221     }
5222
5223     if ($count >= $timeout) {
5224         warn "VM still running - terminating now with SIGKILL\n";
5225         kill 9, $pid;
5226         sleep 1;
5227     }
5228
5229     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5230 }
5231
5232 # Note: use $nocheck to skip tests if VM configuration file exists.
5233 # We need that when migration VMs to other nodes (files already moved)
5234 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5235 sub vm_stop {
5236     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5237
5238     $force = 1 if !defined($force) && !$shutdown;
5239
5240     if ($migratedfrom){
5241         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5242         kill 15, $pid if $pid;
5243         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5244         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5245         return;
5246     }
5247
5248     PVE::QemuConfig->lock_config($vmid, sub {
5249         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5250    });
5251 }
5252
5253 sub vm_reboot {
5254     my ($vmid, $timeout) = @_;
5255
5256     PVE::QemuConfig->lock_config($vmid, sub {
5257         eval {
5258
5259             # only reboot if running, as qmeventd starts it again on a stop event
5260             return if !check_running($vmid);
5261
5262             create_reboot_request($vmid);
5263
5264             my $storecfg = PVE::Storage::config();
5265             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5266
5267         };
5268         if (my $err = $@) {
5269             # avoid that the next normal shutdown will be confused for a reboot
5270             clear_reboot_request($vmid);
5271             die $err;
5272         }
5273    });
5274 }
5275
5276 # note: if using the statestorage parameter, the caller has to check privileges
5277 sub vm_suspend {
5278     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5279
5280     my $conf;
5281     my $path;
5282     my $storecfg;
5283     my $vmstate;
5284
5285     PVE::QemuConfig->lock_config($vmid, sub {
5286
5287         $conf = PVE::QemuConfig->load_config($vmid);
5288
5289         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5290         PVE::QemuConfig->check_lock($conf)
5291             if !($skiplock || $is_backing_up);
5292
5293         die "cannot suspend to disk during backup\n"
5294             if $is_backing_up && $includestate;
5295
5296         if ($includestate) {
5297             $conf->{lock} = 'suspending';
5298             my $date = strftime("%Y-%m-%d", localtime(time()));
5299             $storecfg = PVE::Storage::config();
5300             if (!$statestorage) {
5301                 $statestorage = find_vmstate_storage($conf, $storecfg);
5302                 # check permissions for the storage
5303                 my $rpcenv = PVE::RPCEnvironment::get();
5304                 if ($rpcenv->{type} ne 'cli') {
5305                     my $authuser = $rpcenv->get_user();
5306                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5307                 }
5308             }
5309
5310
5311             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5312             $path = PVE::Storage::path($storecfg, $vmstate);
5313             PVE::QemuConfig->write_config($vmid, $conf);
5314         } else {
5315             mon_cmd($vmid, "stop");
5316         }
5317     });
5318
5319     if ($includestate) {
5320         # save vm state
5321         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5322
5323         eval {
5324             mon_cmd($vmid, "savevm-start", statefile => $path);
5325             for(;;) {
5326                 my $state = mon_cmd($vmid, "query-savevm");
5327                 if (!$state->{status}) {
5328                     die "savevm not active\n";
5329                 } elsif ($state->{status} eq 'active') {
5330                     sleep(1);
5331                     next;
5332                 } elsif ($state->{status} eq 'completed') {
5333                     print "State saved, quitting\n";
5334                     last;
5335                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5336                     die "query-savevm failed with error '$state->{error}'\n"
5337                 } else {
5338                     die "query-savevm returned status '$state->{status}'\n";
5339                 }
5340             }
5341         };
5342         my $err = $@;
5343
5344         PVE::QemuConfig->lock_config($vmid, sub {
5345             $conf = PVE::QemuConfig->load_config($vmid);
5346             if ($err) {
5347                 # cleanup, but leave suspending lock, to indicate something went wrong
5348                 eval {
5349                     mon_cmd($vmid, "savevm-end");
5350                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5351                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5352                     delete $conf->@{qw(vmstate runningmachine)};
5353                     PVE::QemuConfig->write_config($vmid, $conf);
5354                 };
5355                 warn $@ if $@;
5356                 die $err;
5357             }
5358
5359             die "lock changed unexpectedly\n"
5360                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5361
5362             mon_cmd($vmid, "quit");
5363             $conf->{lock} = 'suspended';
5364             PVE::QemuConfig->write_config($vmid, $conf);
5365         });
5366     }
5367 }
5368
5369 sub vm_resume {
5370     my ($vmid, $skiplock, $nocheck) = @_;
5371
5372     PVE::QemuConfig->lock_config($vmid, sub {
5373         my $res = mon_cmd($vmid, 'query-status');
5374         my $resume_cmd = 'cont';
5375
5376         if ($res->{status} && $res->{status} eq 'suspended') {
5377             $resume_cmd = 'system_wakeup';
5378         }
5379
5380         if (!$nocheck) {
5381
5382             my $conf = PVE::QemuConfig->load_config($vmid);
5383
5384             PVE::QemuConfig->check_lock($conf)
5385                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5386         }
5387
5388         mon_cmd($vmid, $resume_cmd);
5389     });
5390 }
5391
5392 sub vm_sendkey {
5393     my ($vmid, $skiplock, $key) = @_;
5394
5395     PVE::QemuConfig->lock_config($vmid, sub {
5396
5397         my $conf = PVE::QemuConfig->load_config($vmid);
5398
5399         # there is no qmp command, so we use the human monitor command
5400         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5401         die $res if $res ne '';
5402     });
5403 }
5404
5405 # vzdump restore implementaion
5406
5407 sub tar_archive_read_firstfile {
5408     my $archive = shift;
5409
5410     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5411
5412     # try to detect archive type first
5413     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5414         die "unable to open file '$archive'\n";
5415     my $firstfile = <$fh>;
5416     kill 15, $pid;
5417     close $fh;
5418
5419     die "ERROR: archive contaions no data\n" if !$firstfile;
5420     chomp $firstfile;
5421
5422     return $firstfile;
5423 }
5424
5425 sub tar_restore_cleanup {
5426     my ($storecfg, $statfile) = @_;
5427
5428     print STDERR "starting cleanup\n";
5429
5430     if (my $fd = IO::File->new($statfile, "r")) {
5431         while (defined(my $line = <$fd>)) {
5432             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5433                 my $volid = $2;
5434                 eval {
5435                     if ($volid =~ m|^/|) {
5436                         unlink $volid || die 'unlink failed\n';
5437                     } else {
5438                         PVE::Storage::vdisk_free($storecfg, $volid);
5439                     }
5440                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5441                 };
5442                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5443             } else {
5444                 print STDERR "unable to parse line in statfile - $line";
5445             }
5446         }
5447         $fd->close();
5448     }
5449 }
5450
5451 sub restore_archive {
5452     my ($archive, $vmid, $user, $opts) = @_;
5453
5454     my $format = $opts->{format};
5455     my $comp;
5456
5457     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5458         $format = 'tar' if !$format;
5459         $comp = 'gzip';
5460     } elsif ($archive =~ m/\.tar$/) {
5461         $format = 'tar' if !$format;
5462     } elsif ($archive =~ m/.tar.lzo$/) {
5463         $format = 'tar' if !$format;
5464         $comp = 'lzop';
5465     } elsif ($archive =~ m/\.vma$/) {
5466         $format = 'vma' if !$format;
5467     } elsif ($archive =~ m/\.vma\.gz$/) {
5468         $format = 'vma' if !$format;
5469         $comp = 'gzip';
5470     } elsif ($archive =~ m/\.vma\.lzo$/) {
5471         $format = 'vma' if !$format;
5472         $comp = 'lzop';
5473     } else {
5474         $format = 'vma' if !$format; # default
5475     }
5476
5477     # try to detect archive format
5478     if ($format eq 'tar') {
5479         return restore_tar_archive($archive, $vmid, $user, $opts);
5480     } else {
5481         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5482     }
5483 }
5484
5485 sub restore_update_config_line {
5486     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5487
5488     return if $line =~ m/^\#qmdump\#/;
5489     return if $line =~ m/^\#vzdump\#/;
5490     return if $line =~ m/^lock:/;
5491     return if $line =~ m/^unused\d+:/;
5492     return if $line =~ m/^parent:/;
5493
5494     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5495     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5496         # try to convert old 1.X settings
5497         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5498         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5499             my ($model, $macaddr) = split(/\=/, $devconfig);
5500             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5501             my $net = {
5502                 model => $model,
5503                 bridge => "vmbr$ind",
5504                 macaddr => $macaddr,
5505             };
5506             my $netstr = print_net($net);
5507
5508             print $outfd "net$cookie->{netcount}: $netstr\n";
5509             $cookie->{netcount}++;
5510         }
5511     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5512         my ($id, $netstr) = ($1, $2);
5513         my $net = parse_net($netstr);
5514         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5515         $netstr = print_net($net);
5516         print $outfd "$id: $netstr\n";
5517     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5518         my $virtdev = $1;
5519         my $value = $3;
5520         my $di = parse_drive($virtdev, $value);
5521         if (defined($di->{backup}) && !$di->{backup}) {
5522             print $outfd "#$line";
5523         } elsif ($map->{$virtdev}) {
5524             delete $di->{format}; # format can change on restore
5525             $di->{file} = $map->{$virtdev};
5526             $value = print_drive($di);
5527             print $outfd "$virtdev: $value\n";
5528         } else {
5529             print $outfd $line;
5530         }
5531     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5532         my $vmgenid = $1;
5533         if ($vmgenid ne '0') {
5534             # always generate a new vmgenid if there was a valid one setup
5535             $vmgenid = generate_uuid();
5536         }
5537         print $outfd "vmgenid: $vmgenid\n";
5538     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5539         my ($uuid, $uuid_str);
5540         UUID::generate($uuid);
5541         UUID::unparse($uuid, $uuid_str);
5542         my $smbios1 = parse_smbios1($2);
5543         $smbios1->{uuid} = $uuid_str;
5544         print $outfd $1.print_smbios1($smbios1)."\n";
5545     } else {
5546         print $outfd $line;
5547     }
5548 }
5549
5550 sub scan_volids {
5551     my ($cfg, $vmid) = @_;
5552
5553     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5554
5555     my $volid_hash = {};
5556     foreach my $storeid (keys %$info) {
5557         foreach my $item (@{$info->{$storeid}}) {
5558             next if !($item->{volid} && $item->{size});
5559             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5560             $volid_hash->{$item->{volid}} = $item;
5561         }
5562     }
5563
5564     return $volid_hash;
5565 }
5566
5567 sub update_disk_config {
5568     my ($vmid, $conf, $volid_hash) = @_;
5569
5570     my $changes;
5571     my $prefix = "VM $vmid:";
5572
5573     # used and unused disks
5574     my $referenced = {};
5575
5576     # Note: it is allowed to define multiple storages with same path (alias), so
5577     # we need to check both 'volid' and real 'path' (two different volid can point
5578     # to the same path).
5579
5580     my $referencedpath = {};
5581
5582     # update size info
5583     foreach my $opt (keys %$conf) {
5584         if (is_valid_drivename($opt)) {
5585             my $drive = parse_drive($opt, $conf->{$opt});
5586             my $volid = $drive->{file};
5587             next if !$volid;
5588
5589             # mark volid as "in-use" for next step
5590             $referenced->{$volid} = 1;
5591             if ($volid_hash->{$volid} &&
5592                 (my $path = $volid_hash->{$volid}->{path})) {
5593                 $referencedpath->{$path} = 1;
5594             }
5595
5596             next if drive_is_cdrom($drive);
5597             next if !$volid_hash->{$volid};
5598
5599             my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5600             if (defined($updated)) {
5601                 $changes = 1;
5602                 $conf->{$opt} = print_drive($updated);
5603                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5604             }
5605         }
5606     }
5607
5608     # remove 'unusedX' entry if volume is used
5609     foreach my $opt (keys %$conf) {
5610         next if $opt !~ m/^unused\d+$/;
5611         my $volid = $conf->{$opt};
5612         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5613         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5614             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5615             $changes = 1;
5616             delete $conf->{$opt};
5617         }
5618
5619         $referenced->{$volid} = 1;
5620         $referencedpath->{$path} = 1 if $path;
5621     }
5622
5623     foreach my $volid (sort keys %$volid_hash) {
5624         next if $volid =~ m/vm-$vmid-state-/;
5625         next if $referenced->{$volid};
5626         my $path = $volid_hash->{$volid}->{path};
5627         next if !$path; # just to be sure
5628         next if $referencedpath->{$path};
5629         $changes = 1;
5630         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5631         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5632         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5633     }
5634
5635     return $changes;
5636 }
5637
5638 sub rescan {
5639     my ($vmid, $nolock, $dryrun) = @_;
5640
5641     my $cfg = PVE::Storage::config();
5642
5643     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5644     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5645     foreach my $stor (keys %{$cfg->{ids}}) {
5646         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5647     }
5648
5649     print "rescan volumes...\n";
5650     my $volid_hash = scan_volids($cfg, $vmid);
5651
5652     my $updatefn =  sub {
5653         my ($vmid) = @_;
5654
5655         my $conf = PVE::QemuConfig->load_config($vmid);
5656
5657         PVE::QemuConfig->check_lock($conf);
5658
5659         my $vm_volids = {};
5660         foreach my $volid (keys %$volid_hash) {
5661             my $info = $volid_hash->{$volid};
5662             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5663         }
5664
5665         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5666
5667         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5668     };
5669
5670     if (defined($vmid)) {
5671         if ($nolock) {
5672             &$updatefn($vmid);
5673         } else {
5674             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5675         }
5676     } else {
5677         my $vmlist = config_list();
5678         foreach my $vmid (keys %$vmlist) {
5679             if ($nolock) {
5680                 &$updatefn($vmid);
5681             } else {
5682                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5683             }
5684         }
5685     }
5686 }
5687
5688 sub restore_vma_archive {
5689     my ($archive, $vmid, $user, $opts, $comp) = @_;
5690
5691     my $readfrom = $archive;
5692
5693     my $cfg = PVE::Storage::config();
5694     my $commands = [];
5695     my $bwlimit = $opts->{bwlimit};
5696
5697     my $dbg_cmdstring = '';
5698     my $add_pipe = sub {
5699         my ($cmd) = @_;
5700         push @$commands, $cmd;
5701         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5702         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5703         $readfrom = '-';
5704     };
5705
5706     my $input = undef;
5707     if ($archive eq '-') {
5708         $input = '<&STDIN';
5709     } else {
5710         # If we use a backup from a PVE defined storage we also consider that
5711         # storage's rate limit:
5712         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5713         if (defined($volid)) {
5714             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5715             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5716             if ($readlimit) {
5717                 print STDERR "applying read rate limit: $readlimit\n";
5718                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5719                 $add_pipe->($cstream);
5720             }
5721         }
5722     }
5723
5724     if ($comp) {
5725         my $cmd;
5726         if ($comp eq 'gzip') {
5727             $cmd = ['zcat', $readfrom];
5728         } elsif ($comp eq 'lzop') {
5729             $cmd = ['lzop', '-d', '-c', $readfrom];
5730         } else {
5731             die "unknown compression method '$comp'\n";
5732         }
5733         $add_pipe->($cmd);
5734     }
5735
5736     my $tmpdir = "/var/tmp/vzdumptmp$$";
5737     rmtree $tmpdir;
5738
5739     # disable interrupts (always do cleanups)
5740     local $SIG{INT} =
5741         local $SIG{TERM} =
5742         local $SIG{QUIT} =
5743         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5744
5745     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5746     POSIX::mkfifo($mapfifo, 0600);
5747     my $fifofh;
5748
5749     my $openfifo = sub {
5750         open($fifofh, '>', $mapfifo) || die $!;
5751     };
5752
5753     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5754
5755     my $oldtimeout;
5756     my $timeout = 5;
5757
5758     my $devinfo = {};
5759
5760     my $rpcenv = PVE::RPCEnvironment::get();
5761
5762     my $conffile = PVE::QemuConfig->config_file($vmid);
5763     my $tmpfn = "$conffile.$$.tmp";
5764
5765     # Note: $oldconf is undef if VM does not exist
5766     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5767     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5768
5769     my %storage_limits;
5770
5771     my $print_devmap = sub {
5772         my $virtdev_hash = {};
5773
5774         my $cfgfn = "$tmpdir/qemu-server.conf";
5775
5776         # we can read the config - that is already extracted
5777         my $fh = IO::File->new($cfgfn, "r") ||
5778             "unable to read qemu-server.conf - $!\n";
5779
5780         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5781         if (-f $fwcfgfn) {
5782             my $pve_firewall_dir = '/etc/pve/firewall';
5783             mkdir $pve_firewall_dir; # make sure the dir exists
5784             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5785         }
5786
5787         while (defined(my $line = <$fh>)) {
5788             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5789                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5790                 die "archive does not contain data for drive '$virtdev'\n"
5791                     if !$devinfo->{$devname};
5792                 if (defined($opts->{storage})) {
5793                     $storeid = $opts->{storage} || 'local';
5794                 } elsif (!$storeid) {
5795                     $storeid = 'local';
5796                 }
5797                 $format = 'raw' if !$format;
5798                 $devinfo->{$devname}->{devname} = $devname;
5799                 $devinfo->{$devname}->{virtdev} = $virtdev;
5800                 $devinfo->{$devname}->{format} = $format;
5801                 $devinfo->{$devname}->{storeid} = $storeid;
5802
5803                 # check permission on storage
5804                 my $pool = $opts->{pool}; # todo: do we need that?
5805                 if ($user ne 'root@pam') {
5806                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5807                 }
5808
5809                 $storage_limits{$storeid} = $bwlimit;
5810
5811                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5812             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5813                 my $virtdev = $1;
5814                 my $drive = parse_drive($virtdev, $2);
5815                 if (drive_is_cloudinit($drive)) {
5816                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5817                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5818                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5819
5820                     my $d = {
5821                         format => $format,
5822                         storeid => $opts->{storage} // $storeid,
5823                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5824                         file => $drive->{file}, # to make drive_is_cloudinit check possible
5825                         name => "vm-$vmid-cloudinit",
5826                         is_cloudinit => 1,
5827                     };
5828                     $virtdev_hash->{$virtdev} = $d;
5829                 }
5830             }
5831         }
5832
5833         foreach my $key (keys %storage_limits) {
5834             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5835             next if !$limit;
5836             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5837             $storage_limits{$key} = $limit * 1024;
5838         }
5839
5840         foreach my $devname (keys %$devinfo) {
5841             die "found no device mapping information for device '$devname'\n"
5842                 if !$devinfo->{$devname}->{virtdev};
5843         }
5844
5845         # create empty/temp config
5846         if ($oldconf) {
5847             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5848             foreach_drive($oldconf, sub {
5849                 my ($ds, $drive) = @_;
5850
5851                 return if drive_is_cdrom($drive, 1);
5852
5853                 my $volid = $drive->{file};
5854                 return if !$volid || $volid =~ m|^/|;
5855
5856                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5857                 return if !$path || !$owner || ($owner != $vmid);
5858
5859                 # Note: only delete disk we want to restore
5860                 # other volumes will become unused
5861                 if ($virtdev_hash->{$ds}) {
5862                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5863                     if (my $err = $@) {
5864                         warn $err;
5865                     }
5866                 }
5867             });
5868
5869             # delete vmstate files, after the restore we have no snapshots anymore
5870             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5871                 my $snap = $oldconf->{snapshots}->{$snapname};
5872                 if ($snap->{vmstate}) {
5873                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5874                     if (my $err = $@) {
5875                         warn $err;
5876                     }
5877                 }
5878             }
5879         }
5880
5881         my $map = {};
5882         foreach my $virtdev (sort keys %$virtdev_hash) {
5883             my $d = $virtdev_hash->{$virtdev};
5884             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5885             my $storeid = $d->{storeid};
5886             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5887
5888             my $map_opts = '';
5889             if (my $limit = $storage_limits{$storeid}) {
5890                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5891             }
5892
5893             # test if requested format is supported
5894             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5895             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5896             $d->{format} = $defFormat if !$supported;
5897
5898             my $name;
5899             if ($d->{is_cloudinit}) {
5900                 $name = $d->{name};
5901                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
5902             }
5903
5904             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5905             print STDERR "new volume ID is '$volid'\n";
5906             $d->{volid} = $volid;
5907
5908             PVE::Storage::activate_volumes($cfg, [$volid]);
5909
5910             my $write_zeros = 1;
5911             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5912                 $write_zeros = 0;
5913             }
5914
5915             if (!$d->{is_cloudinit}) {
5916                 my $path = PVE::Storage::path($cfg, $volid);
5917
5918                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5919
5920                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5921             }
5922             $map->{$virtdev} = $volid;
5923         }
5924
5925         $fh->seek(0, 0) || die "seek failed - $!\n";
5926
5927         my $outfd = new IO::File ($tmpfn, "w") ||
5928             die "unable to write config for VM $vmid\n";
5929
5930         my $cookie = { netcount => 0 };
5931         while (defined(my $line = <$fh>)) {
5932             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5933         }
5934
5935         $fh->close();
5936         $outfd->close();
5937     };
5938
5939     eval {
5940         # enable interrupts
5941         local $SIG{INT} =
5942             local $SIG{TERM} =
5943             local $SIG{QUIT} =
5944             local $SIG{HUP} =
5945             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5946         local $SIG{ALRM} = sub { die "got timeout\n"; };
5947
5948         $oldtimeout = alarm($timeout);
5949
5950         my $parser = sub {
5951             my $line = shift;
5952
5953             print "$line\n";
5954
5955             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5956                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5957                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5958             } elsif ($line =~ m/^CTIME: /) {
5959                 # we correctly received the vma config, so we can disable
5960                 # the timeout now for disk allocation (set to 10 minutes, so
5961                 # that we always timeout if something goes wrong)
5962                 alarm(600);
5963                 &$print_devmap();
5964                 print $fifofh "done\n";
5965                 my $tmp = $oldtimeout || 0;
5966                 $oldtimeout = undef;
5967                 alarm($tmp);
5968                 close($fifofh);
5969             }
5970         };
5971
5972         print "restore vma archive: $dbg_cmdstring\n";
5973         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5974     };
5975     my $err = $@;
5976
5977     alarm($oldtimeout) if $oldtimeout;
5978
5979     my $vollist = [];
5980     foreach my $devname (keys %$devinfo) {
5981         my $volid = $devinfo->{$devname}->{volid};
5982         push @$vollist, $volid if $volid;
5983     }
5984
5985     PVE::Storage::deactivate_volumes($cfg, $vollist);
5986
5987     unlink $mapfifo;
5988
5989     if ($err) {
5990         rmtree $tmpdir;
5991         unlink $tmpfn;
5992
5993         foreach my $devname (keys %$devinfo) {
5994             my $volid = $devinfo->{$devname}->{volid};
5995             next if !$volid;
5996             eval {
5997                 if ($volid =~ m|^/|) {
5998                     unlink $volid || die 'unlink failed\n';
5999                 } else {
6000                     PVE::Storage::vdisk_free($cfg, $volid);
6001                 }
6002                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6003             };
6004             print STDERR "unable to cleanup '$volid' - $@" if $@;
6005         }
6006         die $err;
6007     }
6008
6009     rmtree $tmpdir;
6010
6011     rename($tmpfn, $conffile) ||
6012         die "unable to commit configuration file '$conffile'\n";
6013
6014     PVE::Cluster::cfs_update(); # make sure we read new file
6015
6016     eval { rescan($vmid, 1); };
6017     warn $@ if $@;
6018 }
6019
6020 sub restore_tar_archive {
6021     my ($archive, $vmid, $user, $opts) = @_;
6022
6023     if ($archive ne '-') {
6024         my $firstfile = tar_archive_read_firstfile($archive);
6025         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6026             if $firstfile ne 'qemu-server.conf';
6027     }
6028
6029     my $storecfg = PVE::Storage::config();
6030
6031     # avoid zombie disks when restoring over an existing VM -> cleanup first
6032     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6033     # skiplock=1 because qmrestore has set the 'create' lock itself already
6034     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6035     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6036
6037     my $tocmd = "/usr/lib/qemu-server/qmextract";
6038
6039     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6040     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6041     $tocmd .= ' --prealloc' if $opts->{prealloc};
6042     $tocmd .= ' --info' if $opts->{info};
6043
6044     # tar option "xf" does not autodetect compression when read from STDIN,
6045     # so we pipe to zcat
6046     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6047         PVE::Tools::shellquote("--to-command=$tocmd");
6048
6049     my $tmpdir = "/var/tmp/vzdumptmp$$";
6050     mkpath $tmpdir;
6051
6052     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6053     local $ENV{VZDUMP_VMID} = $vmid;
6054     local $ENV{VZDUMP_USER} = $user;
6055
6056     my $conffile = PVE::QemuConfig->config_file($vmid);
6057     my $tmpfn = "$conffile.$$.tmp";
6058
6059     # disable interrupts (always do cleanups)
6060     local $SIG{INT} =
6061         local $SIG{TERM} =
6062         local $SIG{QUIT} =
6063         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6064
6065     eval {
6066         # enable interrupts
6067         local $SIG{INT} =
6068             local $SIG{TERM} =
6069             local $SIG{QUIT} =
6070             local $SIG{HUP} =
6071             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6072
6073         if ($archive eq '-') {
6074             print "extracting archive from STDIN\n";
6075             run_command($cmd, input => "<&STDIN");
6076         } else {
6077             print "extracting archive '$archive'\n";
6078             run_command($cmd);
6079         }
6080
6081         return if $opts->{info};
6082
6083         # read new mapping
6084         my $map = {};
6085         my $statfile = "$tmpdir/qmrestore.stat";
6086         if (my $fd = IO::File->new($statfile, "r")) {
6087             while (defined (my $line = <$fd>)) {
6088                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6089                     $map->{$1} = $2 if $1;
6090                 } else {
6091                     print STDERR "unable to parse line in statfile - $line\n";
6092                 }
6093             }
6094             $fd->close();
6095         }
6096
6097         my $confsrc = "$tmpdir/qemu-server.conf";
6098
6099         my $srcfd = new IO::File($confsrc, "r") ||
6100             die "unable to open file '$confsrc'\n";
6101
6102         my $outfd = new IO::File ($tmpfn, "w") ||
6103             die "unable to write config for VM $vmid\n";
6104
6105         my $cookie = { netcount => 0 };
6106         while (defined (my $line = <$srcfd>)) {
6107             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6108         }
6109
6110         $srcfd->close();
6111         $outfd->close();
6112     };
6113     if (my $err = $@) {
6114         unlink $tmpfn;
6115         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6116         die $err;
6117     }
6118
6119     rmtree $tmpdir;
6120
6121     rename $tmpfn, $conffile ||
6122         die "unable to commit configuration file '$conffile'\n";
6123
6124     PVE::Cluster::cfs_update(); # make sure we read new file
6125
6126     eval { rescan($vmid, 1); };
6127     warn $@ if $@;
6128 };
6129
6130 sub foreach_storage_used_by_vm {
6131     my ($conf, $func) = @_;
6132
6133     my $sidhash = {};
6134
6135     foreach_drive($conf, sub {
6136         my ($ds, $drive) = @_;
6137         return if drive_is_cdrom($drive);
6138
6139         my $volid = $drive->{file};
6140
6141         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6142         $sidhash->{$sid} = $sid if $sid;
6143     });
6144
6145     foreach my $sid (sort keys %$sidhash) {
6146         &$func($sid);
6147     }
6148 }
6149
6150 my $qemu_snap_storage = {
6151     rbd => 1,
6152 };
6153 sub do_snapshots_with_qemu {
6154     my ($storecfg, $volid) = @_;
6155
6156     my $storage_name = PVE::Storage::parse_volume_id($volid);
6157     my $scfg = $storecfg->{ids}->{$storage_name};
6158
6159     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6160         return 1;
6161     }
6162
6163     if ($volid =~ m/\.(qcow2|qed)$/){
6164         return 1;
6165     }
6166
6167     return undef;
6168 }
6169
6170 sub qga_check_running {
6171     my ($vmid, $nowarn) = @_;
6172
6173     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6174     if ($@) {
6175         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6176         return 0;
6177     }
6178     return 1;
6179 }
6180
6181 sub template_create {
6182     my ($vmid, $conf, $disk) = @_;
6183
6184     my $storecfg = PVE::Storage::config();
6185
6186     foreach_drive($conf, sub {
6187         my ($ds, $drive) = @_;
6188
6189         return if drive_is_cdrom($drive);
6190         return if $disk && $ds ne $disk;
6191
6192         my $volid = $drive->{file};
6193         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6194
6195         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6196         $drive->{file} = $voliddst;
6197         $conf->{$ds} = print_drive($drive);
6198         PVE::QemuConfig->write_config($vmid, $conf);
6199     });
6200 }
6201
6202 sub convert_iscsi_path {
6203     my ($path) = @_;
6204
6205     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6206         my $portal = $1;
6207         my $target = $2;
6208         my $lun = $3;
6209
6210         my $initiator_name = get_initiator_name();
6211
6212         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6213                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6214     }
6215
6216     die "cannot convert iscsi path '$path', unkown format\n";
6217 }
6218
6219 sub qemu_img_convert {
6220     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6221
6222     my $storecfg = PVE::Storage::config();
6223     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6224     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6225
6226     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6227
6228     my $cachemode;
6229     my $src_path;
6230     my $src_is_iscsi = 0;
6231     my $src_format;
6232
6233     if ($src_storeid) {
6234         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6235         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6236         $src_format = qemu_img_format($src_scfg, $src_volname);
6237         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6238         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6239         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6240     } elsif (-f $src_volid) {
6241         $src_path = $src_volid;
6242         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6243             $src_format = $1;
6244         }
6245     }
6246
6247     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6248
6249     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6250     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6251     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6252     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6253
6254     my $cmd = [];
6255     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6256     push @$cmd, '-l', "snapshot.name=$snapname"
6257         if $snapname && $src_format && $src_format eq "qcow2";
6258     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6259     push @$cmd, '-T', $cachemode if defined($cachemode);
6260
6261     if ($src_is_iscsi) {
6262         push @$cmd, '--image-opts';
6263         $src_path = convert_iscsi_path($src_path);
6264     } elsif ($src_format) {
6265         push @$cmd, '-f', $src_format;
6266     }
6267
6268     if ($dst_is_iscsi) {
6269         push @$cmd, '--target-image-opts';
6270         $dst_path = convert_iscsi_path($dst_path);
6271     } else {
6272         push @$cmd, '-O', $dst_format;
6273     }
6274
6275     push @$cmd, $src_path;
6276
6277     if (!$dst_is_iscsi && $is_zero_initialized) {
6278         push @$cmd, "zeroinit:$dst_path";
6279     } else {
6280         push @$cmd, $dst_path;
6281     }
6282
6283     my $parser = sub {
6284         my $line = shift;
6285         if($line =~ m/\((\S+)\/100\%\)/){
6286             my $percent = $1;
6287             my $transferred = int($size * $percent / 100);
6288             my $remaining = $size - $transferred;
6289
6290             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6291         }
6292
6293     };
6294
6295     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6296     my $err = $@;
6297     die "copy failed: $err" if $err;
6298 }
6299
6300 sub qemu_img_format {
6301     my ($scfg, $volname) = @_;
6302
6303     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6304         return $1;
6305     } else {
6306         return "raw";
6307     }
6308 }
6309
6310 sub qemu_drive_mirror {
6311     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6312
6313     $jobs = {} if !$jobs;
6314
6315     my $qemu_target;
6316     my $format;
6317     $jobs->{"drive-$drive"} = {};
6318
6319     if ($dst_volid =~ /^nbd:/) {
6320         $qemu_target = $dst_volid;
6321         $format = "nbd";
6322     } else {
6323         my $storecfg = PVE::Storage::config();
6324         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6325
6326         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6327
6328         $format = qemu_img_format($dst_scfg, $dst_volname);
6329
6330         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6331
6332         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6333     }
6334
6335     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6336     $opts->{format} = $format if $format;
6337
6338     if (defined($bwlimit)) {
6339         $opts->{speed} = $bwlimit * 1024;
6340         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6341     } else {
6342         print "drive mirror is starting for drive-$drive\n";
6343     }
6344
6345     # if a job already runs for this device we get an error, catch it for cleanup
6346     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6347     if (my $err = $@) {
6348         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6349         warn "$@\n" if $@;
6350         die "mirroring error: $err\n";
6351     }
6352
6353     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6354 }
6355
6356 sub qemu_drive_mirror_monitor {
6357     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6358
6359     eval {
6360         my $err_complete = 0;
6361
6362         while (1) {
6363             die "storage migration timed out\n" if $err_complete > 300;
6364
6365             my $stats = mon_cmd($vmid, "query-block-jobs");
6366
6367             my $running_mirror_jobs = {};
6368             foreach my $stat (@$stats) {
6369                 next if $stat->{type} ne 'mirror';
6370                 $running_mirror_jobs->{$stat->{device}} = $stat;
6371             }
6372
6373             my $readycounter = 0;
6374
6375             foreach my $job (keys %$jobs) {
6376
6377                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6378                     print "$job : finished\n";
6379                     delete $jobs->{$job};
6380                     next;
6381                 }
6382
6383                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6384
6385                 my $busy = $running_mirror_jobs->{$job}->{busy};
6386                 my $ready = $running_mirror_jobs->{$job}->{ready};
6387                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6388                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6389                     my $remaining = $total - $transferred;
6390                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6391
6392                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6393                 }
6394
6395                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6396             }
6397
6398             last if scalar(keys %$jobs) == 0;
6399
6400             if ($readycounter == scalar(keys %$jobs)) {
6401                 print "all mirroring jobs are ready \n";
6402                 last if $skipcomplete; #do the complete later
6403
6404                 if ($vmiddst && $vmiddst != $vmid) {
6405                     my $agent_running = $qga && qga_check_running($vmid);
6406                     if ($agent_running) {
6407                         print "freeze filesystem\n";
6408                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6409                     } else {
6410                         print "suspend vm\n";
6411                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6412                     }
6413
6414                     # if we clone a disk for a new target vm, we don't switch the disk
6415                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6416
6417                     if ($agent_running) {
6418                         print "unfreeze filesystem\n";
6419                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6420                     } else {
6421                         print "resume vm\n";
6422                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6423                     }
6424
6425                     last;
6426                 } else {
6427
6428                     foreach my $job (keys %$jobs) {
6429                         # try to switch the disk if source and destination are on the same guest
6430                         print "$job: Completing block job...\n";
6431
6432                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6433                         if ($@ =~ m/cannot be completed/) {
6434                             print "$job: Block job cannot be completed, try again.\n";
6435                             $err_complete++;
6436                         }else {
6437                             print "$job: Completed successfully.\n";
6438                             $jobs->{$job}->{complete} = 1;
6439                         }
6440                     }
6441                 }
6442             }
6443             sleep 1;
6444         }
6445     };
6446     my $err = $@;
6447
6448     if ($err) {
6449         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6450         die "mirroring error: $err";
6451     }
6452
6453 }
6454
6455 sub qemu_blockjobs_cancel {
6456     my ($vmid, $jobs) = @_;
6457
6458     foreach my $job (keys %$jobs) {
6459         print "$job: Cancelling block job\n";
6460         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6461         $jobs->{$job}->{cancel} = 1;
6462     }
6463
6464     while (1) {
6465         my $stats = mon_cmd($vmid, "query-block-jobs");
6466
6467         my $running_jobs = {};
6468         foreach my $stat (@$stats) {
6469             $running_jobs->{$stat->{device}} = $stat;
6470         }
6471
6472         foreach my $job (keys %$jobs) {
6473
6474             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6475                 print "$job: Done.\n";
6476                 delete $jobs->{$job};
6477             }
6478         }
6479
6480         last if scalar(keys %$jobs) == 0;
6481
6482         sleep 1;
6483     }
6484 }
6485
6486 sub clone_disk {
6487     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6488         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6489
6490     my $newvolid;
6491
6492     if (!$full) {
6493         print "create linked clone of drive $drivename ($drive->{file})\n";
6494         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6495         push @$newvollist, $newvolid;
6496     } else {
6497
6498         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6499         $storeid = $storage if $storage;
6500
6501         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6502         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6503
6504         print "create full clone of drive $drivename ($drive->{file})\n";
6505         my $name = undef;
6506         if (drive_is_cloudinit($drive)) {
6507             $name = "vm-$newvmid-cloudinit";
6508             $name .= ".$dst_format" if $dst_format ne 'raw';
6509             $snapname = undef;
6510             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6511         }
6512         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6513         push @$newvollist, $newvolid;
6514
6515         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6516
6517         if (drive_is_cloudinit($drive)) {
6518             goto no_data_clone;
6519         }
6520
6521         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6522         if (!$running || $snapname) {
6523             # TODO: handle bwlimits
6524             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6525         } else {
6526
6527             my $kvmver = get_running_qemu_version ($vmid);
6528             if (!min_version($kvmver, 2, 7)) {
6529                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6530                     if $drive->{iothread};
6531             }
6532
6533             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6534         }
6535     }
6536
6537 no_data_clone:
6538     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6539
6540     my $disk = $drive;
6541     $disk->{format} = undef;
6542     $disk->{file} = $newvolid;
6543     $disk->{size} = $size;
6544
6545     return $disk;
6546 }
6547
6548 sub get_running_qemu_version {
6549     my ($vmid) = @_;
6550     my $res = mon_cmd($vmid, "query-version");
6551     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6552 }
6553
6554 sub qemu_use_old_bios_files {
6555     my ($machine_type) = @_;
6556
6557     return if !$machine_type;
6558
6559     my $use_old_bios_files = undef;
6560
6561     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6562         $machine_type = $1;
6563         $use_old_bios_files = 1;
6564     } else {
6565         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6566         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6567         # load new efi bios files on migration. So this hack is required to allow
6568         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6569         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6570         $use_old_bios_files = !min_version($version, 2, 4);
6571     }
6572
6573     return ($use_old_bios_files, $machine_type);
6574 }
6575
6576 sub create_efidisk($$$$$) {
6577     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6578
6579     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6580     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6581
6582     my $vars_size_b = -s $ovmf_vars;
6583     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6584     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6585     PVE::Storage::activate_volumes($storecfg, [$volid]);
6586
6587     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6588     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6589
6590     return ($volid, $size/1024);
6591 }
6592
6593 sub vm_iothreads_list {
6594     my ($vmid) = @_;
6595
6596     my $res = mon_cmd($vmid, 'query-iothreads');
6597
6598     my $iothreads = {};
6599     foreach my $iothread (@$res) {
6600         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6601     }
6602
6603     return $iothreads;
6604 }
6605
6606 sub scsihw_infos {
6607     my ($conf, $drive) = @_;
6608
6609     my $maxdev = 0;
6610
6611     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6612         $maxdev = 7;
6613     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6614         $maxdev = 1;
6615     } else {
6616         $maxdev = 256;
6617     }
6618
6619     my $controller = int($drive->{index} / $maxdev);
6620     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6621
6622     return ($maxdev, $controller, $controller_prefix);
6623 }
6624
6625 sub windows_version {
6626     my ($ostype) = @_;
6627
6628     return 0 if !$ostype;
6629
6630     my $winversion = 0;
6631
6632     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6633         $winversion = 5;
6634     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6635         $winversion = 6;
6636     } elsif ($ostype =~ m/^win(\d+)$/) {
6637         $winversion = $1;
6638     }
6639
6640     return $winversion;
6641 }
6642
6643 sub resolve_dst_disk_format {
6644         my ($storecfg, $storeid, $src_volname, $format) = @_;
6645         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6646
6647         if (!$format) {
6648             # if no target format is specified, use the source disk format as hint
6649             if ($src_volname) {
6650                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6651                 $format = qemu_img_format($scfg, $src_volname);
6652             } else {
6653                 return $defFormat;
6654             }
6655         }
6656
6657         # test if requested format is supported - else use default
6658         my $supported = grep { $_ eq $format } @$validFormats;
6659         $format = $defFormat if !$supported;
6660         return $format;
6661 }
6662
6663 # NOTE: if this logic changes, please update docs & possibly gui logic
6664 sub find_vmstate_storage {
6665     my ($conf, $storecfg) = @_;
6666
6667     # first, return storage from conf if set
6668     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
6669
6670     my ($target, $shared, $local);
6671
6672     foreach_storage_used_by_vm($conf, sub {
6673         my ($sid) = @_;
6674         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
6675         my $dst = $scfg->{shared} ? \$shared : \$local;
6676         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
6677     });
6678
6679     # second, use shared storage where VM has at least one disk
6680     # third, use local storage where VM has at least one disk
6681     # fall back to local storage
6682     $target = $shared // $local // 'local';
6683
6684     return $target;
6685 }
6686
6687 sub generate_uuid {
6688     my ($uuid, $uuid_str);
6689     UUID::generate($uuid);
6690     UUID::unparse($uuid, $uuid_str);
6691     return $uuid_str;
6692 }
6693
6694 sub generate_smbios1_uuid {
6695     return "uuid=".generate_uuid();
6696 }
6697
6698 sub nbd_stop {
6699     my ($vmid) = @_;
6700
6701     mon_cmd($vmid, 'nbd-server-stop');
6702 }
6703
6704 sub create_reboot_request {
6705     my ($vmid) = @_;
6706     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
6707         or die "failed to create reboot trigger file: $!\n";
6708     close($fh);
6709 }
6710
6711 sub clear_reboot_request {
6712     my ($vmid) = @_;
6713     my $path = "/run/qemu-server/$vmid.reboot";
6714     my $res = 0;
6715
6716     $res = unlink($path);
6717     die "could not remove reboot request for $vmid: $!"
6718         if !$res && $! != POSIX::ENOENT;
6719
6720     return $res;
6721 }
6722
6723 # bash completion helper
6724
6725 sub complete_backup_archives {
6726     my ($cmdname, $pname, $cvalue) = @_;
6727
6728     my $cfg = PVE::Storage::config();
6729
6730     my $storeid;
6731
6732     if ($cvalue =~ m/^([^:]+):/) {
6733         $storeid = $1;
6734     }
6735
6736     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6737
6738     my $res = [];
6739     foreach my $id (keys %$data) {
6740         foreach my $item (@{$data->{$id}}) {
6741             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6742             push @$res, $item->{volid} if defined($item->{volid});
6743         }
6744     }
6745
6746     return $res;
6747 }
6748
6749 my $complete_vmid_full = sub {
6750     my ($running) = @_;
6751
6752     my $idlist = vmstatus();
6753
6754     my $res = [];
6755
6756     foreach my $id (keys %$idlist) {
6757         my $d = $idlist->{$id};
6758         if (defined($running)) {
6759             next if $d->{template};
6760             next if $running && $d->{status} ne 'running';
6761             next if !$running && $d->{status} eq 'running';
6762         }
6763         push @$res, $id;
6764
6765     }
6766     return $res;
6767 };
6768
6769 sub complete_vmid {
6770     return &$complete_vmid_full();
6771 }
6772
6773 sub complete_vmid_stopped {
6774     return &$complete_vmid_full(0);
6775 }
6776
6777 sub complete_vmid_running {
6778     return &$complete_vmid_full(1);
6779 }
6780
6781 sub complete_storage {
6782
6783     my $cfg = PVE::Storage::config();
6784     my $ids = $cfg->{ids};
6785
6786     my $res = [];
6787     foreach my $sid (keys %$ids) {
6788         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6789         next if !$ids->{$sid}->{content}->{images};
6790         push @$res, $sid;
6791     }
6792
6793     return $res;
6794 }
6795
6796 sub complete_migration_storage {
6797     my ($cmd, $param, $current_value, $all_args) = @_;
6798
6799     my $targetnode = @$all_args[1];
6800
6801     my $cfg = PVE::Storage::config();
6802     my $ids = $cfg->{ids};
6803
6804     my $res = [];
6805     foreach my $sid (keys %$ids) {
6806         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
6807         next if !$ids->{$sid}->{content}->{images};
6808         push @$res, $sid;
6809     }
6810
6811     return $res;
6812 }
6813
6814 1;