PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016/2019
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 970 dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $usb_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string', format => 'pve-qm-usb-device',
1005         format_description => 'HOSTUSBDEVICE|spice',
1006         description => <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1008
1009  'bus-port(.port)*' (decimal numbers) or
1010  'vendor_id:product_id' (hexadeciaml numbers) or
1011  'spice'
1012
1013 You can use the 'lsusb -t' command to list existing usb devices.
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1017
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1019 EODESCR
1020     },
1021     usb3 => {
1022         optional => 1,
1023         type => 'boolean',
1024         description => "Specifies whether if given host option is a USB3 device or port.",
1025         default => 0,
1026     },
1027 };
1028
1029 my $usbdesc = {
1030     optional => 1,
1031     type => 'string', format => $usb_fmt,
1032     description => "Configure an USB device (n is 0 to 4).",
1033 };
1034 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1035
1036 my $serialdesc = {
1037         optional => 1,
1038         type => 'string',
1039         pattern => '(/dev/.+|socket)',
1040         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1041         verbose_description =>  <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1045
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052
1053 my $paralleldesc= {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1057         description =>  "Map host parallel devices (n is 0 to 2).",
1058         verbose_description =>  <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1078 }
1079
1080 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1081     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1082 }
1083
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1085     $confdesc->{"usb$i"} = $usbdesc;
1086 }
1087
1088 my $boot_fmt = {
1089     legacy => {
1090         optional => 1,
1091         default_key => 1,
1092         type => 'string',
1093         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094                      . " Deprecated, use 'order=' instead.",
1095         pattern => '[acdn]{1,4}',
1096         format_description => "[acdn]{1,4}",
1097
1098         # note: this is also the fallback if boot: is not given at all
1099         default => 'cdn',
1100     },
1101     order => {
1102         optional => 1,
1103         type => 'string',
1104         format => 'pve-qm-bootdev-list',
1105         format_description => "device[;device...]",
1106         description => <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1108
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1112
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1116
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1118 EODESC
1119     },
1120 };
1121 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1122
1123 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1124 sub verify_bootdev {
1125     my ($dev, $noerr) = @_;
1126
1127     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1128
1129     my $check = sub {
1130         my ($base) = @_;
1131         return 0 if $dev !~ m/^$base\d+$/;
1132         return 0 if !$confdesc->{$dev};
1133         return 1;
1134     };
1135
1136     return $dev if $check->("net");
1137     return $dev if $check->("usb");
1138     return $dev if $check->("hostpci");
1139
1140     return if $noerr;
1141     die "invalid boot device '$dev'\n";
1142 }
1143
1144 sub print_bootorder {
1145     my ($devs) = @_;
1146     return "" if !@$devs;
1147     my $data = { order => join(';', @$devs) };
1148     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1149 }
1150
1151 my $kvm_api_version = 0;
1152
1153 sub kvm_version {
1154     return $kvm_api_version if $kvm_api_version;
1155
1156     open my $fh, '<', '/dev/kvm' or return;
1157
1158     # 0xae00 => KVM_GET_API_VERSION
1159     $kvm_api_version = ioctl($fh, 0xae00, 0);
1160     close($fh);
1161
1162     return $kvm_api_version;
1163 }
1164
1165 my $kvm_user_version = {};
1166 my $kvm_mtime = {};
1167
1168 sub kvm_user_version {
1169     my ($binary) = @_;
1170
1171     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1172     my $st = stat($binary);
1173
1174     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176         $cachedmtime == $st->mtime;
1177
1178     $kvm_user_version->{$binary} = 'unknown';
1179     $kvm_mtime->{$binary} = $st->mtime;
1180
1181     my $code = sub {
1182         my $line = shift;
1183         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184             $kvm_user_version->{$binary} = $2;
1185         }
1186     };
1187
1188     eval { run_command([$binary, '--version'], outfunc => $code); };
1189     warn $@ if $@;
1190
1191     return $kvm_user_version->{$binary};
1192
1193 }
1194 my sub extract_version {
1195     my ($machine_type, $version) = @_;
1196     $version = kvm_user_version() if !defined($version);
1197     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1198 }
1199
1200 sub kernel_has_vhost_net {
1201     return -c '/dev/vhost-net';
1202 }
1203
1204 sub option_exists {
1205     my $key = shift;
1206     return defined($confdesc->{$key});
1207 }
1208
1209 my $cdrom_path;
1210 sub get_cdrom_path {
1211
1212     return  $cdrom_path if $cdrom_path;
1213
1214     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1215     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1216     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1217 }
1218
1219 sub get_iso_path {
1220     my ($storecfg, $vmid, $cdrom) = @_;
1221
1222     if ($cdrom eq 'cdrom') {
1223         return get_cdrom_path();
1224     } elsif ($cdrom eq 'none') {
1225         return '';
1226     } elsif ($cdrom =~ m|^/|) {
1227         return $cdrom;
1228     } else {
1229         return PVE::Storage::path($storecfg, $cdrom);
1230     }
1231 }
1232
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id {
1235     my ($vmid, $file, $media) = @_;
1236
1237      if (!($file eq 'none' || $file eq 'cdrom' ||
1238           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1239
1240         return if $file =~ m|/|;
1241
1242         if ($media && $media eq 'cdrom') {
1243             $file = "local:iso/$file";
1244         } else {
1245             $file = "local:$vmid/$file";
1246         }
1247     }
1248
1249     return $file;
1250 }
1251
1252 sub verify_media_type {
1253     my ($opt, $vtype, $media) = @_;
1254
1255     return if !$media;
1256
1257     my $etype;
1258     if ($media eq 'disk') {
1259         $etype = 'images';
1260     } elsif ($media eq 'cdrom') {
1261         $etype = 'iso';
1262     } else {
1263         die "internal error";
1264     }
1265
1266     return if ($vtype eq $etype);
1267
1268     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1269 }
1270
1271 sub cleanup_drive_path {
1272     my ($opt, $storecfg, $drive) = @_;
1273
1274     # try to convert filesystem paths to volume IDs
1275
1276     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1277         ($drive->{file} !~ m|^/dev/.+|) &&
1278         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1279         ($drive->{file} !~ m/^\d+$/)) {
1280         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1281         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1282             if !$vtype;
1283         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1284         verify_media_type($opt, $vtype, $drive->{media});
1285         $drive->{file} = $volid;
1286     }
1287
1288     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1289 }
1290
1291 sub parse_hotplug_features {
1292     my ($data) = @_;
1293
1294     my $res = {};
1295
1296     return $res if $data eq '0';
1297
1298     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1299
1300     foreach my $feature (PVE::Tools::split_list($data)) {
1301         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1302             $res->{$1} = 1;
1303         } else {
1304             die "invalid hotplug feature '$feature'\n";
1305         }
1306     }
1307     return $res;
1308 }
1309
1310 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1311 sub pve_verify_hotplug_features {
1312     my ($value, $noerr) = @_;
1313
1314     return $value if parse_hotplug_features($value);
1315
1316     return if $noerr;
1317
1318     die "unable to parse hotplug option\n";
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub print_tabletdevice_full {
1383     my ($conf, $arch) = @_;
1384
1385     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1386
1387     # we use uhci for old VMs because tablet driver was buggy in older qemu
1388     my $usbbus;
1389     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1390         $usbbus = 'ehci';
1391     } else {
1392         $usbbus = 'uhci';
1393     }
1394
1395     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1396 }
1397
1398 sub print_keyboarddevice_full {
1399     my ($conf, $arch, $machine) = @_;
1400
1401     return if $arch ne 'aarch64';
1402
1403     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1404 }
1405
1406 sub print_drivedevice_full {
1407     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1408
1409     my $device = '';
1410     my $maxdev = 0;
1411
1412     my $drive_id = "$drive->{interface}$drive->{index}";
1413     if ($drive->{interface} eq 'virtio') {
1414         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1415         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1417     } elsif ($drive->{interface} eq 'scsi') {
1418
1419         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1420         my $unit = $drive->{index} % $maxdev;
1421         my $devicetype = 'hd';
1422         my $path = '';
1423         if (drive_is_cdrom($drive)) {
1424             $devicetype = 'cd';
1425         } else {
1426             if ($drive->{file} =~ m|^/|) {
1427                 $path = $drive->{file};
1428                 if (my $info = path_is_scsi($path)) {
1429                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1430                         $devicetype = 'block';
1431                     } elsif ($info->{type} == 1) { # tape
1432                         $devicetype = 'generic';
1433                     }
1434                 }
1435             } else {
1436                  $path = PVE::Storage::path($storecfg, $drive->{file});
1437             }
1438
1439             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440             my $version = extract_version($machine_type, kvm_user_version());
1441             if ($path =~ m/^iscsi\:\/\// &&
1442                !min_version($version, 4, 1)) {
1443                 $devicetype = 'generic';
1444             }
1445         }
1446
1447         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1448             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1449         } else {
1450             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451                 .",lun=$drive->{index}";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456             $device .= ",rotation_rate=1";
1457         }
1458         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1459
1460     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1461         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1462         my $controller = int($drive->{index} / $maxdev);
1463         my $unit = $drive->{index} % $maxdev;
1464         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1465
1466         $device = "ide-$devicetype";
1467         if ($drive->{interface} eq 'ide') {
1468             $device .= ",bus=ide.$controller,unit=$unit";
1469         } else {
1470             $device .= ",bus=ahci$controller.$unit";
1471         }
1472         $device .= ",drive=drive-$drive_id,id=$drive_id";
1473
1474         if ($devicetype eq 'hd') {
1475             if (my $model = $drive->{model}) {
1476                 $model = URI::Escape::uri_unescape($model);
1477                 $device .= ",model=$model";
1478             }
1479             if ($drive->{ssd}) {
1480                 $device .= ",rotation_rate=1";
1481             }
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484     } elsif ($drive->{interface} eq 'usb') {
1485         die "implement me";
1486         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1487     } else {
1488         die "unsupported interface type";
1489     }
1490
1491     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1492
1493     if (my $serial = $drive->{serial}) {
1494         $serial = URI::Escape::uri_unescape($serial);
1495         $device .= ",serial=$serial";
1496     }
1497
1498
1499     return $device;
1500 }
1501
1502 sub get_initiator_name {
1503     my $initiator;
1504
1505     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1506     while (defined(my $line = <$fh>)) {
1507         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1508         $initiator = $1;
1509         last;
1510     }
1511     $fh->close();
1512
1513     return $initiator;
1514 }
1515
1516 sub print_drive_commandline_full {
1517     my ($storecfg, $vmid, $drive) = @_;
1518
1519     my $path;
1520     my $volid = $drive->{file};
1521     my $format;
1522
1523     if (drive_is_cdrom($drive)) {
1524         $path = get_iso_path($storecfg, $vmid, $volid);
1525     } else {
1526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1527         if ($storeid) {
1528             $path = PVE::Storage::path($storecfg, $volid);
1529             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1530             $format = qemu_img_format($scfg, $volname);
1531         } else {
1532             $path = $volid;
1533             $format = "raw";
1534         }
1535    }
1536
1537     my $opts = '';
1538     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539     foreach my $o (@qemu_drive_options) {
1540         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1541     }
1542
1543     # snapshot only accepts on|off
1544     if (defined($drive->{snapshot})) {
1545         my $v = $drive->{snapshot} ? 'on' : 'off';
1546         $opts .= ",snapshot=$v";
1547     }
1548
1549     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1550         my ($dir, $qmpname) = @$type;
1551         if (my $v = $drive->{"mbps$dir"}) {
1552             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1553         }
1554         if (my $v = $drive->{"mbps${dir}_max"}) {
1555             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1556         }
1557         if (my $v = $drive->{"bps${dir}_max_length"}) {
1558             $opts .= ",throttling.bps$qmpname-max-length=$v";
1559         }
1560         if (my $v = $drive->{"iops${dir}"}) {
1561             $opts .= ",throttling.iops$qmpname=$v";
1562         }
1563         if (my $v = $drive->{"iops${dir}_max"}) {
1564             $opts .= ",throttling.iops$qmpname-max=$v";
1565         }
1566         if (my $v = $drive->{"iops${dir}_max_length"}) {
1567             $opts .= ",throttling.iops$qmpname-max-length=$v";
1568         }
1569     }
1570
1571     $opts .= ",format=$format" if $format && !$drive->{format};
1572
1573     my $cache_direct = 0;
1574
1575     if (my $cache = $drive->{cache}) {
1576         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577     } elsif (!drive_is_cdrom($drive)) {
1578         $opts .= ",cache=none";
1579         $cache_direct = 1;
1580     }
1581
1582     # aio native works only with O_DIRECT
1583     if (!$drive->{aio}) {
1584         if($cache_direct) {
1585             $opts .= ",aio=native";
1586         } else {
1587             $opts .= ",aio=threads";
1588         }
1589     }
1590
1591     if (!drive_is_cdrom($drive)) {
1592         my $detectzeroes;
1593         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1594             $detectzeroes = 'off';
1595         } elsif ($drive->{discard}) {
1596             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1597         } else {
1598             # This used to be our default with discard not being specified:
1599             $detectzeroes = 'on';
1600         }
1601         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1602     }
1603
1604     my $pathinfo = $path ? "file=$path," : '';
1605
1606     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1607 }
1608
1609 sub print_netdevice_full {
1610     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1611
1612     my $device = $net->{model};
1613     if ($net->{model} eq 'virtio') {
1614          $device = 'virtio-net-pci';
1615      };
1616
1617     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1618     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1620         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621         # and out of each queue plus one config interrupt and control vector queue
1622         my $vectors = $net->{queues} * 2 + 2;
1623         $tmpstr .= ",vectors=$vectors,mq=on";
1624     }
1625     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1626
1627     if (my $mtu = $net->{mtu}) {
1628         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1629             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1630             if ($mtu == 1) {
1631                  $mtu = $bridge_mtu;
1632             } elsif ($mtu < 576) {
1633                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634             } elsif ($mtu > $bridge_mtu) {
1635                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1636             }
1637             $tmpstr .= ",host_mtu=$mtu";
1638         } else {
1639             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1640         }
1641     }
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     if (is_native($arch)) {
1680         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1681     }
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1691     } else {
1692         $netdev = "type=user,id=$netid,hostname=$vmname";
1693     }
1694
1695     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1696
1697     return $netdev;
1698 }
1699
1700 my $vga_map = {
1701     'cirrus' => 'cirrus-vga',
1702     'std' => 'VGA',
1703     'vmware' => 'vmware-svga',
1704     'virtio' => 'virtio-vga',
1705 };
1706
1707 sub print_vga_device {
1708     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1709
1710     my $type = $vga_map->{$vga->{type}};
1711     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712         $type = 'virtio-gpu';
1713     }
1714     my $vgamem_mb = $vga->{memory};
1715
1716     my $max_outputs = '';
1717     if ($qxlnum) {
1718         $type = $id ? 'qxl' : 'qxl-vga';
1719
1720         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1721             # set max outputs so linux can have up to 4 qxl displays with one device
1722             if (min_version($machine_version, 4, 1)) {
1723                 $max_outputs = ",max_outputs=4";
1724             }
1725         }
1726     }
1727
1728     die "no devicetype for $vga->{type}\n" if !$type;
1729
1730     my $memory = "";
1731     if ($vgamem_mb) {
1732         if ($vga->{type} eq 'virtio') {
1733             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1734             $memory = ",max_hostmem=$bytes";
1735         } elsif ($qxlnum) {
1736             # from https://www.spice-space.org/multiple-monitors.html
1737             $memory = ",vgamem_mb=$vga->{memory}";
1738             my $ram = $vgamem_mb * 4;
1739             my $vram = $vgamem_mb * 2;
1740             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1741         } else {
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743         }
1744     } elsif ($qxlnum && $id) {
1745         $memory = ",ram_size=67108864,vram_size=33554432";
1746     }
1747
1748     my $edidoff = "";
1749     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1750         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1751     }
1752
1753     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1754     my $vgaid = "vga" . ($id // '');
1755     my $pciaddr;
1756
1757     if ($q35 && $vgaid eq 'vga') {
1758         # the first display uses pcie.0 bus on q35 machines
1759         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1760     } else {
1761         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1762     }
1763
1764     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1765 }
1766
1767 sub parse_number_sets {
1768     my ($set) = @_;
1769     my $res = [];
1770     foreach my $part (split(/;/, $set)) {
1771         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773             push @$res, [ $1, $2 ];
1774         } else {
1775             die "invalid range: $part\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 sub parse_numa {
1782     my ($data) = @_;
1783
1784     my $res = parse_property_string($numa_fmt, $data);
1785     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1786     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1787     return $res;
1788 }
1789
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1791 sub parse_net {
1792     my ($data) = @_;
1793
1794     my $res = eval { parse_property_string($net_fmt, $data) };
1795     if ($@) {
1796         warn $@;
1797         return;
1798     }
1799     if (!defined($res->{macaddr})) {
1800         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1801         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1802     }
1803     return $res;
1804 }
1805
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig {
1808     my ($data) = @_;
1809
1810     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1811     if ($@) {
1812         warn $@;
1813         return;
1814     }
1815
1816     if ($res->{gw} && !$res->{ip}) {
1817         warn 'gateway specified without specifying an IP address';
1818         return;
1819     }
1820     if ($res->{gw6} && !$res->{ip6}) {
1821         warn 'IPv6 gateway specified without specifying an IPv6 address';
1822         return;
1823     }
1824     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1825         warn 'gateway specified together with DHCP';
1826         return;
1827     }
1828     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1829         # gw6 + auto/dhcp
1830         warn "IPv6 gateway specified together with $res->{ip6} address";
1831         return;
1832     }
1833
1834     if (!$res->{ip} && !$res->{ip6}) {
1835         return { ip => 'dhcp', ip6 => 'dhcp' };
1836     }
1837
1838     return $res;
1839 }
1840
1841 sub print_net {
1842     my $net = shift;
1843
1844     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1845 }
1846
1847 sub add_random_macs {
1848     my ($settings) = @_;
1849
1850     foreach my $opt (keys %$settings) {
1851         next if $opt !~ m/^net(\d+)$/;
1852         my $net = parse_net($settings->{$opt});
1853         next if !$net;
1854         $settings->{$opt} = print_net($net);
1855     }
1856 }
1857
1858 sub vm_is_volid_owner {
1859     my ($storecfg, $vmid, $volid) = @_;
1860
1861     if ($volid !~  m|^/|) {
1862         my ($path, $owner);
1863         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1864         if ($owner && ($owner == $vmid)) {
1865             return 1;
1866         }
1867     }
1868
1869     return;
1870 }
1871
1872 sub vmconfig_register_unused_drive {
1873     my ($storecfg, $vmid, $conf, $drive) = @_;
1874
1875     if (drive_is_cloudinit($drive)) {
1876         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1877         warn $@ if $@;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         my $volid = $drive->{file};
1880         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1881             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1882         }
1883     }
1884 }
1885
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1887 my $smbios1_fmt = {
1888     uuid => {
1889         type => 'string',
1890         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891         format_description => 'UUID',
1892         description => "Set SMBIOS1 UUID.",
1893         optional => 1,
1894     },
1895     version => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 version.",
1900         optional => 1,
1901     },
1902     serial => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 serial number.",
1907         optional => 1,
1908     },
1909     manufacturer => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 manufacturer.",
1914         optional => 1,
1915     },
1916     product => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 product ID.",
1921         optional => 1,
1922     },
1923     sku => {
1924         type => 'string',
1925         pattern => '[A-Za-z0-9+\/]+={0,2}',
1926         format_description => 'Base64 encoded string',
1927         description => "Set SMBIOS1 SKU string.",
1928         optional => 1,
1929     },
1930     family => {
1931         type => 'string',
1932         pattern => '[A-Za-z0-9+\/]+={0,2}',
1933         format_description => 'Base64 encoded string',
1934         description => "Set SMBIOS1 family string.",
1935         optional => 1,
1936     },
1937     base64 => {
1938         type => 'boolean',
1939         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1940         optional => 1,
1941     },
1942 };
1943
1944 sub parse_smbios1 {
1945     my ($data) = @_;
1946
1947     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1948     warn $@ if $@;
1949     return $res;
1950 }
1951
1952 sub print_smbios1 {
1953     my ($smbios1) = @_;
1954     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1958
1959 sub parse_watchdog {
1960     my ($value) = @_;
1961
1962     return if !$value;
1963
1964     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1965     warn $@ if $@;
1966     return $res;
1967 }
1968
1969 sub parse_guest_agent {
1970     my ($conf) = @_;
1971
1972     return {} if !defined($conf->{agent});
1973
1974     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1975     warn $@ if $@;
1976
1977     # if the agent is disabled ignore the other potentially set properties
1978     return {} if !$res->{enabled};
1979     return $res;
1980 }
1981
1982 sub get_qga_key {
1983     my ($conf, $key) = @_;
1984     return undef if !defined($conf->{agent});
1985
1986     my $agent = parse_guest_agent($conf);
1987     return $agent->{$key};
1988 }
1989
1990 sub parse_vga {
1991     my ($value) = @_;
1992
1993     return {} if !$value;
1994     my $res = eval { parse_property_string($vga_fmt, $value) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub parse_rng {
2000     my ($value) = @_;
2001
2002     return if !$value;
2003
2004     my $res = eval { parse_property_string($rng_fmt, $value) };
2005     warn $@ if $@;
2006     return $res;
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2010 sub verify_usb_device {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if parse_usb_device($value);
2014
2015     return if $noerr;
2016
2017     die "unable to parse usb device\n";
2018 }
2019
2020 # add JSON properties for create and set function
2021 sub json_config_properties {
2022     my $prop = shift;
2023
2024     foreach my $opt (keys %$confdesc) {
2025         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027         $prop->{$opt} = $confdesc->{$opt};
2028     }
2029
2030     return $prop;
2031 }
2032
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties {
2035
2036     return dclone($confdesc_cloudinit);
2037 }
2038
2039 sub check_type {
2040     my ($key, $value) = @_;
2041
2042     die "unknown setting '$key'\n" if !$confdesc->{$key};
2043
2044     my $type = $confdesc->{$key}->{type};
2045
2046     if (!defined($value)) {
2047         die "got undefined value\n";
2048     }
2049
2050     if ($value =~ m/[\n\r]/) {
2051         die "property contains a line feed\n";
2052     }
2053
2054     if ($type eq 'boolean') {
2055         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057         die "type check ('boolean') failed - got '$value'\n";
2058     } elsif ($type eq 'integer') {
2059         return int($1) if $value =~ m/^(\d+)$/;
2060         die "type check ('integer') failed - got '$value'\n";
2061     } elsif ($type eq 'number') {
2062         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063         die "type check ('number') failed - got '$value'\n";
2064     } elsif ($type eq 'string') {
2065         if (my $fmt = $confdesc->{$key}->{format}) {
2066             PVE::JSONSchema::check_format($fmt, $value);
2067             return $value;
2068         }
2069         $value =~ s/^\"(.*)\"$/$1/;
2070         return $value;
2071     } else {
2072         die "internal error"
2073     }
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     if ($conf->{template}) {
2084         # check if any base image is still used by a linked clone
2085         PVE::QemuConfig->foreach_volume($conf, sub {
2086                 my ($ds, $drive) = @_;
2087                 return if drive_is_cdrom($drive);
2088
2089                 my $volid = $drive->{file};
2090                 return if !$volid || $volid =~ m|^/|;
2091
2092                 die "base volume '$volid' is still in use by linked cloned\n"
2093                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2094
2095         });
2096     }
2097
2098     # only remove disks owned by this VM (referenced in the config)
2099     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2100         my ($ds, $drive) = @_;
2101         return if drive_is_cdrom($drive, 1);
2102
2103         my $volid = $drive->{file};
2104         return if !$volid || $volid =~ m|^/|;
2105
2106         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2107         return if !$path || !$owner || ($owner != $vmid);
2108
2109         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2110         warn "Could not remove disk '$volid', check manually: $@" if $@;
2111     });
2112
2113     if ($purge_unreferenced) { # also remove unreferenced disk
2114         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2115         PVE::Storage::foreach_volid($vmdisks, sub {
2116             my ($volid, $sid, $volname, $d) = @_;
2117             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2118             warn $@ if $@;
2119         });
2120     }
2121
2122     if (defined $replacement_conf) {
2123         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2124     } else {
2125         PVE::QemuConfig->destroy_config($vmid);
2126     }
2127 }
2128
2129 sub parse_vm_config {
2130     my ($filename, $raw) = @_;
2131
2132     return if !defined($raw);
2133
2134     my $res = {
2135         digest => Digest::SHA::sha1_hex($raw),
2136         snapshots => {},
2137         pending => {},
2138     };
2139
2140     $filename =~ m|/qemu-server/(\d+)\.conf$|
2141         || die "got strange filename '$filename'";
2142
2143     my $vmid = $1;
2144
2145     my $conf = $res;
2146     my $descr;
2147     my $section = '';
2148
2149     my @lines = split(/\n/, $raw);
2150     foreach my $line (@lines) {
2151         next if $line =~ m/^\s*$/;
2152
2153         if ($line =~ m/^\[PENDING\]\s*$/i) {
2154             $section = 'pending';
2155             if (defined($descr)) {
2156                 $descr =~ s/\s+$//;
2157                 $conf->{description} = $descr;
2158             }
2159             $descr = undef;
2160             $conf = $res->{$section} = {};
2161             next;
2162
2163         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2164             $section = $1;
2165             if (defined($descr)) {
2166                 $descr =~ s/\s+$//;
2167                 $conf->{description} = $descr;
2168             }
2169             $descr = undef;
2170             $conf = $res->{snapshots}->{$section} = {};
2171             next;
2172         }
2173
2174         if ($line =~ m/^\#(.*)\s*$/) {
2175             $descr = '' if !defined($descr);
2176             $descr .= PVE::Tools::decode_text($1) . "\n";
2177             next;
2178         }
2179
2180         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181             $descr = '' if !defined($descr);
2182             $descr .= PVE::Tools::decode_text($2);
2183         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184             $conf->{snapstate} = $1;
2185         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2186             my $key = $1;
2187             my $value = $2;
2188             $conf->{$key} = $value;
2189         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2190             my $value = $1;
2191             if ($section eq 'pending') {
2192                 $conf->{delete} = $value; # we parse this later
2193             } else {
2194                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2195             }
2196         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2197             my $key = $1;
2198             my $value = $2;
2199             eval { $value = check_type($key, $value); };
2200             if ($@) {
2201                 warn "vm $vmid - unable to parse value of '$key' - $@";
2202             } else {
2203                 $key = 'ide2' if $key eq 'cdrom';
2204                 my $fmt = $confdesc->{$key}->{format};
2205                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206                     my $v = parse_drive($key, $value);
2207                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2208                         $v->{file} = $volid;
2209                         $value = print_drive($v);
2210                     } else {
2211                         warn "vm $vmid - unable to parse value of '$key'\n";
2212                         next;
2213                     }
2214                 }
2215
2216                 $conf->{$key} = $value;
2217             }
2218         } else {
2219             warn "vm $vmid - unable to parse config: $line\n";
2220         }
2221     }
2222
2223     if (defined($descr)) {
2224         $descr =~ s/\s+$//;
2225         $conf->{description} = $descr;
2226     }
2227     delete $res->{snapstate}; # just to be sure
2228
2229     return $res;
2230 }
2231
2232 sub write_vm_config {
2233     my ($filename, $conf) = @_;
2234
2235     delete $conf->{snapstate}; # just to be sure
2236
2237     if ($conf->{cdrom}) {
2238         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2239         $conf->{ide2} = $conf->{cdrom};
2240         delete $conf->{cdrom};
2241     }
2242
2243     # we do not use 'smp' any longer
2244     if ($conf->{sockets}) {
2245         delete $conf->{smp};
2246     } elsif ($conf->{smp}) {
2247         $conf->{sockets} = $conf->{smp};
2248         delete $conf->{cores};
2249         delete $conf->{smp};
2250     }
2251
2252     my $used_volids = {};
2253
2254     my $cleanup_config = sub {
2255         my ($cref, $pending, $snapname) = @_;
2256
2257         foreach my $key (keys %$cref) {
2258             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2259                 $key eq 'snapstate' || $key eq 'pending';
2260             my $value = $cref->{$key};
2261             if ($key eq 'delete') {
2262                 die "propertry 'delete' is only allowed in [PENDING]\n"
2263                     if !$pending;
2264                 # fixme: check syntax?
2265                 next;
2266             }
2267             eval { $value = check_type($key, $value); };
2268             die "unable to parse value of '$key' - $@" if $@;
2269
2270             $cref->{$key} = $value;
2271
2272             if (!$snapname && is_valid_drivename($key)) {
2273                 my $drive = parse_drive($key, $value);
2274                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2275             }
2276         }
2277     };
2278
2279     &$cleanup_config($conf);
2280
2281     &$cleanup_config($conf->{pending}, 1);
2282
2283     foreach my $snapname (keys %{$conf->{snapshots}}) {
2284         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2285         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2286     }
2287
2288     # remove 'unusedX' settings if we re-add a volume
2289     foreach my $key (keys %$conf) {
2290         my $value = $conf->{$key};
2291         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2292             delete $conf->{$key};
2293         }
2294     }
2295
2296     my $generate_raw_config = sub {
2297         my ($conf, $pending) = @_;
2298
2299         my $raw = '';
2300
2301         # add description as comment to top of file
2302         if (defined(my $descr = $conf->{description})) {
2303             if ($descr) {
2304                 foreach my $cl (split(/\n/, $descr)) {
2305                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2306                 }
2307             } else {
2308                 $raw .= "#\n" if $pending;
2309             }
2310         }
2311
2312         foreach my $key (sort keys %$conf) {
2313             next if $key =~ /^(digest|description|pending|snapshots)$/;
2314             $raw .= "$key: $conf->{$key}\n";
2315         }
2316         return $raw;
2317     };
2318
2319     my $raw = &$generate_raw_config($conf);
2320
2321     if (scalar(keys %{$conf->{pending}})){
2322         $raw .= "\n[PENDING]\n";
2323         $raw .= &$generate_raw_config($conf->{pending}, 1);
2324     }
2325
2326     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2327         $raw .= "\n[$snapname]\n";
2328         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2329     }
2330
2331     return $raw;
2332 }
2333
2334 sub load_defaults {
2335
2336     my $res = {};
2337
2338     # we use static defaults from our JSON schema configuration
2339     foreach my $key (keys %$confdesc) {
2340         if (defined(my $default = $confdesc->{$key}->{default})) {
2341             $res->{$key} = $default;
2342         }
2343     }
2344
2345     return $res;
2346 }
2347
2348 sub config_list {
2349     my $vmlist = PVE::Cluster::get_vmlist();
2350     my $res = {};
2351     return $res if !$vmlist || !$vmlist->{ids};
2352     my $ids = $vmlist->{ids};
2353     my $nodename = nodename();
2354
2355     foreach my $vmid (keys %$ids) {
2356         my $d = $ids->{$vmid};
2357         next if !$d->{node} || $d->{node} ne $nodename;
2358         next if !$d->{type} || $d->{type} ne 'qemu';
2359         $res->{$vmid}->{exists} = 1;
2360     }
2361     return $res;
2362 }
2363
2364 # test if VM uses local resources (to prevent migration)
2365 sub check_local_resources {
2366     my ($conf, $noerr) = @_;
2367
2368     my @loc_res = ();
2369
2370     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2371     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2372
2373     push @loc_res, "ivshmem" if $conf->{ivshmem};
2374
2375     foreach my $k (keys %$conf) {
2376         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2377         # sockets are safe: they will recreated be on the target side post-migrate
2378         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2379         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2380     }
2381
2382     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2383
2384     return \@loc_res;
2385 }
2386
2387 # check if used storages are available on all nodes (use by migrate)
2388 sub check_storage_availability {
2389     my ($storecfg, $conf, $node) = @_;
2390
2391     PVE::QemuConfig->foreach_volume($conf, sub {
2392         my ($ds, $drive) = @_;
2393
2394         my $volid = $drive->{file};
2395         return if !$volid;
2396
2397         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2398         return if !$sid;
2399
2400         # check if storage is available on both nodes
2401         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2402         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2403    });
2404 }
2405
2406 # list nodes where all VM images are available (used by has_feature API)
2407 sub shared_nodes {
2408     my ($conf, $storecfg) = @_;
2409
2410     my $nodelist = PVE::Cluster::get_nodelist();
2411     my $nodehash = { map { $_ => 1 } @$nodelist };
2412     my $nodename = nodename();
2413
2414     PVE::QemuConfig->foreach_volume($conf, sub {
2415         my ($ds, $drive) = @_;
2416
2417         my $volid = $drive->{file};
2418         return if !$volid;
2419
2420         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2421         if ($storeid) {
2422             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2423             if ($scfg->{disable}) {
2424                 $nodehash = {};
2425             } elsif (my $avail = $scfg->{nodes}) {
2426                 foreach my $node (keys %$nodehash) {
2427                     delete $nodehash->{$node} if !$avail->{$node};
2428                 }
2429             } elsif (!$scfg->{shared}) {
2430                 foreach my $node (keys %$nodehash) {
2431                     delete $nodehash->{$node} if $node ne $nodename
2432                 }
2433             }
2434         }
2435     });
2436
2437     return $nodehash
2438 }
2439
2440 sub check_local_storage_availability {
2441     my ($conf, $storecfg) = @_;
2442
2443     my $nodelist = PVE::Cluster::get_nodelist();
2444     my $nodehash = { map { $_ => {} } @$nodelist };
2445
2446     PVE::QemuConfig->foreach_volume($conf, sub {
2447         my ($ds, $drive) = @_;
2448
2449         my $volid = $drive->{file};
2450         return if !$volid;
2451
2452         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2453         if ($storeid) {
2454             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2455
2456             if ($scfg->{disable}) {
2457                 foreach my $node (keys %$nodehash) {
2458                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2459                 }
2460             } elsif (my $avail = $scfg->{nodes}) {
2461                 foreach my $node (keys %$nodehash) {
2462                     if (!$avail->{$node}) {
2463                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2464                     }
2465                 }
2466             }
2467         }
2468     });
2469
2470     foreach my $node (values %$nodehash) {
2471         if (my $unavail = $node->{unavailable_storages}) {
2472             $node->{unavailable_storages} = [ sort keys %$unavail ];
2473         }
2474     }
2475
2476     return $nodehash
2477 }
2478
2479 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2480 sub check_running {
2481     my ($vmid, $nocheck, $node) = @_;
2482
2483     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2484     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2485 }
2486
2487 sub vzlist {
2488
2489     my $vzlist = config_list();
2490
2491     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2492
2493     while (defined(my $de = $fd->read)) {
2494         next if $de !~ m/^(\d+)\.pid$/;
2495         my $vmid = $1;
2496         next if !defined($vzlist->{$vmid});
2497         if (my $pid = check_running($vmid)) {
2498             $vzlist->{$vmid}->{pid} = $pid;
2499         }
2500     }
2501
2502     return $vzlist;
2503 }
2504
2505 our $vmstatus_return_properties = {
2506     vmid => get_standard_option('pve-vmid'),
2507     status => {
2508         description => "Qemu process status.",
2509         type => 'string',
2510         enum => ['stopped', 'running'],
2511     },
2512     maxmem => {
2513         description => "Maximum memory in bytes.",
2514         type => 'integer',
2515         optional => 1,
2516         renderer => 'bytes',
2517     },
2518     maxdisk => {
2519         description => "Root disk size in bytes.",
2520         type => 'integer',
2521         optional => 1,
2522         renderer => 'bytes',
2523     },
2524     name => {
2525         description => "VM name.",
2526         type => 'string',
2527         optional => 1,
2528     },
2529     qmpstatus => {
2530         description => "Qemu QMP agent status.",
2531         type => 'string',
2532         optional => 1,
2533     },
2534     pid => {
2535         description => "PID of running qemu process.",
2536         type => 'integer',
2537         optional => 1,
2538     },
2539     uptime => {
2540         description => "Uptime.",
2541         type => 'integer',
2542         optional => 1,
2543         renderer => 'duration',
2544     },
2545     cpus => {
2546         description => "Maximum usable CPUs.",
2547         type => 'number',
2548         optional => 1,
2549     },
2550     lock => {
2551         description => "The current config lock, if any.",
2552         type => 'string',
2553         optional => 1,
2554     },
2555     tags => {
2556         description  => "The current configured tags, if any",
2557         type => 'string',
2558         optional => 1,
2559     },
2560     'running-machine' => {
2561         description => "The currently running machine type (if running).",
2562         type => 'string',
2563         optional => 1,
2564     },
2565     'running-qemu' => {
2566         description => "The currently running QEMU version (if running).",
2567         type => 'string',
2568         optional => 1,
2569     },
2570 };
2571
2572 my $last_proc_pid_stat;
2573
2574 # get VM status information
2575 # This must be fast and should not block ($full == false)
2576 # We only query KVM using QMP if $full == true (this can be slow)
2577 sub vmstatus {
2578     my ($opt_vmid, $full) = @_;
2579
2580     my $res = {};
2581
2582     my $storecfg = PVE::Storage::config();
2583
2584     my $list = vzlist();
2585     my $defaults = load_defaults();
2586
2587     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2588
2589     my $cpucount = $cpuinfo->{cpus} || 1;
2590
2591     foreach my $vmid (keys %$list) {
2592         next if $opt_vmid && ($vmid ne $opt_vmid);
2593
2594         my $conf = PVE::QemuConfig->load_config($vmid);
2595
2596         my $d = { vmid => $vmid };
2597         $d->{pid} = $list->{$vmid}->{pid};
2598
2599         # fixme: better status?
2600         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2601
2602         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2603         if (defined($size)) {
2604             $d->{disk} = 0; # no info available
2605             $d->{maxdisk} = $size;
2606         } else {
2607             $d->{disk} = 0;
2608             $d->{maxdisk} = 0;
2609         }
2610
2611         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2612             * ($conf->{cores} || $defaults->{cores});
2613         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2614         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2615
2616         $d->{name} = $conf->{name} || "VM $vmid";
2617         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2618             : $defaults->{memory}*(1024*1024);
2619
2620         if ($conf->{balloon}) {
2621             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2622             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2623                 : $defaults->{shares};
2624         }
2625
2626         $d->{uptime} = 0;
2627         $d->{cpu} = 0;
2628         $d->{mem} = 0;
2629
2630         $d->{netout} = 0;
2631         $d->{netin} = 0;
2632
2633         $d->{diskread} = 0;
2634         $d->{diskwrite} = 0;
2635
2636         $d->{template} = PVE::QemuConfig->is_template($conf);
2637
2638         $d->{serial} = 1 if conf_has_serial($conf);
2639         $d->{lock} = $conf->{lock} if $conf->{lock};
2640         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2641
2642         $res->{$vmid} = $d;
2643     }
2644
2645     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2646     foreach my $dev (keys %$netdev) {
2647         next if $dev !~ m/^tap([1-9]\d*)i/;
2648         my $vmid = $1;
2649         my $d = $res->{$vmid};
2650         next if !$d;
2651
2652         $d->{netout} += $netdev->{$dev}->{receive};
2653         $d->{netin} += $netdev->{$dev}->{transmit};
2654
2655         if ($full) {
2656             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2657             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2658         }
2659
2660     }
2661
2662     my $ctime = gettimeofday;
2663
2664     foreach my $vmid (keys %$list) {
2665
2666         my $d = $res->{$vmid};
2667         my $pid = $d->{pid};
2668         next if !$pid;
2669
2670         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2671         next if !$pstat; # not running
2672
2673         my $used = $pstat->{utime} + $pstat->{stime};
2674
2675         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2676
2677         if ($pstat->{vsize}) {
2678             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2679         }
2680
2681         my $old = $last_proc_pid_stat->{$pid};
2682         if (!$old) {
2683             $last_proc_pid_stat->{$pid} = {
2684                 time => $ctime,
2685                 used => $used,
2686                 cpu => 0,
2687             };
2688             next;
2689         }
2690
2691         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2692
2693         if ($dtime > 1000) {
2694             my $dutime = $used -  $old->{used};
2695
2696             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2697             $last_proc_pid_stat->{$pid} = {
2698                 time => $ctime,
2699                 used => $used,
2700                 cpu => $d->{cpu},
2701             };
2702         } else {
2703             $d->{cpu} = $old->{cpu};
2704         }
2705     }
2706
2707     return $res if !$full;
2708
2709     my $qmpclient = PVE::QMPClient->new();
2710
2711     my $ballooncb = sub {
2712         my ($vmid, $resp) = @_;
2713
2714         my $info = $resp->{'return'};
2715         return if !$info->{max_mem};
2716
2717         my $d = $res->{$vmid};
2718
2719         # use memory assigned to VM
2720         $d->{maxmem} = $info->{max_mem};
2721         $d->{balloon} = $info->{actual};
2722
2723         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2724             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2725             $d->{freemem} = $info->{free_mem};
2726         }
2727
2728         $d->{ballooninfo} = $info;
2729     };
2730
2731     my $blockstatscb = sub {
2732         my ($vmid, $resp) = @_;
2733         my $data = $resp->{'return'} || [];
2734         my $totalrdbytes = 0;
2735         my $totalwrbytes = 0;
2736
2737         for my $blockstat (@$data) {
2738             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2739             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2740
2741             $blockstat->{device} =~ s/drive-//;
2742             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2743         }
2744         $res->{$vmid}->{diskread} = $totalrdbytes;
2745         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2746     };
2747
2748     my $machinecb = sub {
2749         my ($vmid, $resp) = @_;
2750         my $data = $resp->{'return'} || [];
2751
2752         $res->{$vmid}->{'running-machine'} =
2753             PVE::QemuServer::Machine::current_from_query_machines($data);
2754     };
2755
2756     my $versioncb = sub {
2757         my ($vmid, $resp) = @_;
2758         my $data = $resp->{'return'} // {};
2759         my $version = 'unknown';
2760
2761         if (my $v = $data->{qemu}) {
2762             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2763         }
2764
2765         $res->{$vmid}->{'running-qemu'} = $version;
2766     };
2767
2768     my $statuscb = sub {
2769         my ($vmid, $resp) = @_;
2770
2771         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2772         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2773         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2774         # this fails if ballon driver is not loaded, so this must be
2775         # the last commnand (following command are aborted if this fails).
2776         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2777
2778         my $status = 'unknown';
2779         if (!defined($status = $resp->{'return'}->{status})) {
2780             warn "unable to get VM status\n";
2781             return;
2782         }
2783
2784         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2785     };
2786
2787     foreach my $vmid (keys %$list) {
2788         next if $opt_vmid && ($vmid ne $opt_vmid);
2789         next if !$res->{$vmid}->{pid}; # not running
2790         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2791     }
2792
2793     $qmpclient->queue_execute(undef, 2);
2794
2795     foreach my $vmid (keys %$list) {
2796         next if $opt_vmid && ($vmid ne $opt_vmid);
2797         next if !$res->{$vmid}->{pid}; #not running
2798
2799         # we can't use the $qmpclient since it might have already aborted on
2800         # 'query-balloon', but this might also fail for older versions...
2801         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2802         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2803     }
2804
2805     foreach my $vmid (keys %$list) {
2806         next if $opt_vmid && ($vmid ne $opt_vmid);
2807         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2808     }
2809
2810     return $res;
2811 }
2812
2813 sub conf_has_serial {
2814     my ($conf) = @_;
2815
2816     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2817         if ($conf->{"serial$i"}) {
2818             return 1;
2819         }
2820     }
2821
2822     return 0;
2823 }
2824
2825 sub conf_has_audio {
2826     my ($conf, $id) = @_;
2827
2828     $id //= 0;
2829     my $audio = $conf->{"audio$id"};
2830     return if !defined($audio);
2831
2832     my $audioproperties = parse_property_string($audio_fmt, $audio);
2833     my $audiodriver = $audioproperties->{driver} // 'spice';
2834
2835     return {
2836         dev => $audioproperties->{device},
2837         dev_id => "audiodev$id",
2838         backend => $audiodriver,
2839         backend_id => "$audiodriver-backend${id}",
2840     };
2841 }
2842
2843 sub audio_devs {
2844     my ($audio, $audiopciaddr, $machine_version) = @_;
2845
2846     my $devs = [];
2847
2848     my $id = $audio->{dev_id};
2849     my $audiodev = "";
2850     if (min_version($machine_version, 4, 2)) {
2851         $audiodev = ",audiodev=$audio->{backend_id}";
2852     }
2853
2854     if ($audio->{dev} eq 'AC97') {
2855         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2856     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2857         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2858         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2859         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2860     } else {
2861         die "unkown audio device '$audio->{dev}', implement me!";
2862     }
2863
2864     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2865
2866     return $devs;
2867 }
2868
2869 sub vga_conf_has_spice {
2870     my ($vga) = @_;
2871
2872     my $vgaconf = parse_vga($vga);
2873     my $vgatype = $vgaconf->{type};
2874     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2875
2876     return $1 || 1;
2877 }
2878
2879 sub is_native($) {
2880     my ($arch) = @_;
2881     return get_host_arch() eq $arch;
2882 }
2883
2884 sub get_vm_arch {
2885     my ($conf) = @_;
2886     return $conf->{arch} // get_host_arch();
2887 }
2888
2889 my $default_machines = {
2890     x86_64 => 'pc',
2891     aarch64 => 'virt',
2892 };
2893
2894 sub get_installed_machine_version {
2895     my ($kvmversion) = @_;
2896     $kvmversion = kvm_user_version() if !defined($kvmversion);
2897     $kvmversion =~ m/^(\d+\.\d+)/;
2898     return $1;
2899 }
2900
2901 sub windows_get_pinned_machine_version {
2902     my ($machine, $base_version, $kvmversion) = @_;
2903
2904     my $pin_version = $base_version;
2905     if (!defined($base_version) ||
2906         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2907     ) {
2908         $pin_version = get_installed_machine_version($kvmversion);
2909     }
2910     if (!$machine || $machine eq 'pc') {
2911         $machine = "pc-i440fx-$pin_version";
2912     } elsif ($machine eq 'q35') {
2913         $machine = "pc-q35-$pin_version";
2914     } elsif ($machine eq 'virt') {
2915         $machine = "virt-$pin_version";
2916     } else {
2917         warn "unknown machine type '$machine', not touching that!\n";
2918     }
2919
2920     return $machine;
2921 }
2922
2923 sub get_vm_machine {
2924     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2925
2926     my $machine = $forcemachine || $conf->{machine};
2927
2928     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2929         $kvmversion //= kvm_user_version();
2930         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2931         # layout which confuses windows quite a bit and may result in various regressions..
2932         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2933         if (windows_version($conf->{ostype})) {
2934             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2935         }
2936         $arch //= 'x86_64';
2937         $machine ||= $default_machines->{$arch};
2938         if ($add_pve_version) {
2939             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2940             $machine .= "+pve$pvever";
2941         }
2942     }
2943
2944     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2945         # for version-pinned machines that do not include a pve-version (e.g.
2946         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2947         $machine .= '+pve0';
2948     }
2949
2950     return $machine;
2951 }
2952
2953 sub get_ovmf_files($) {
2954     my ($arch) = @_;
2955
2956     my $ovmf = $OVMF->{$arch}
2957         or die "no OVMF images known for architecture '$arch'\n";
2958
2959     return @$ovmf;
2960 }
2961
2962 my $Arch2Qemu = {
2963     aarch64 => '/usr/bin/qemu-system-aarch64',
2964     x86_64 => '/usr/bin/qemu-system-x86_64',
2965 };
2966 sub get_command_for_arch($) {
2967     my ($arch) = @_;
2968     return '/usr/bin/kvm' if is_native($arch);
2969
2970     my $cmd = $Arch2Qemu->{$arch}
2971         or die "don't know how to emulate architecture '$arch'\n";
2972     return $cmd;
2973 }
2974
2975 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2976 # to use in a QEMU command line (-cpu element), first array_intersect the result
2977 # of query_supported_ with query_understood_. This is necessary because:
2978 #
2979 # a) query_understood_ returns flags the host cannot use and
2980 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2981 #    flags, but CPU settings - with most of them being flags. Those settings
2982 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2983 #
2984 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2985 # expensive. If you need the value returned from this, you can get it much
2986 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2987 # $accel being 'kvm' or 'tcg'.
2988 #
2989 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2990 # changes, automatically populating pmxcfs.
2991 #
2992 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2993 # since kvm and tcg machines support different flags
2994 #
2995 sub query_supported_cpu_flags {
2996     my ($arch) = @_;
2997
2998     $arch //= get_host_arch();
2999     my $default_machine = $default_machines->{$arch};
3000
3001     my $flags = {};
3002
3003     # FIXME: Once this is merged, the code below should work for ARM as well:
3004     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3005     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3006         $arch eq "aarch64";
3007
3008     my $kvm_supported = defined(kvm_version());
3009     my $qemu_cmd = get_command_for_arch($arch);
3010     my $fakevmid = -1;
3011     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3012
3013     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3014     my $query_supported_run_qemu = sub {
3015         my ($kvm) = @_;
3016
3017         my $flags = {};
3018         my $cmd = [
3019             $qemu_cmd,
3020             '-machine', $default_machine,
3021             '-display', 'none',
3022             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3023             '-mon', 'chardev=qmp,mode=control',
3024             '-pidfile', $pidfile,
3025             '-S', '-daemonize'
3026         ];
3027
3028         if (!$kvm) {
3029             push @$cmd, '-accel', 'tcg';
3030         }
3031
3032         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3033         die "QEMU flag querying VM exited with code " . $rc if $rc;
3034
3035         eval {
3036             my $cmd_result = mon_cmd(
3037                 $fakevmid,
3038                 'query-cpu-model-expansion',
3039                 type => 'full',
3040                 model => { name => 'host' }
3041             );
3042
3043             my $props = $cmd_result->{model}->{props};
3044             foreach my $prop (keys %$props) {
3045                 next if $props->{$prop} ne '1';
3046                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3047                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3048                 # We only keep those with underscores, to match /proc/cpuinfo
3049                 $prop =~ s/\.|-/_/g;
3050                 $flags->{$prop} = 1;
3051             }
3052         };
3053         my $err = $@;
3054
3055         # force stop with 10 sec timeout and 'nocheck'
3056         # always stop, even if QMP failed
3057         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3058
3059         die $err if $err;
3060
3061         return [ sort keys %$flags ];
3062     };
3063
3064     # We need to query QEMU twice, since KVM and TCG have different supported flags
3065     PVE::QemuConfig->lock_config($fakevmid, sub {
3066         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3067         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3068
3069         if ($kvm_supported) {
3070             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3071             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3072         }
3073     });
3074
3075     return $flags;
3076 }
3077
3078 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3079 my $understood_cpu_flag_dir = "/usr/share/kvm";
3080 sub query_understood_cpu_flags {
3081     my $arch = get_host_arch();
3082     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3083
3084     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3085         if ! -e $filepath;
3086
3087     my $raw = file_get_contents($filepath);
3088     $raw =~ s/^\s+|\s+$//g;
3089     my @flags = split(/\s+/, $raw);
3090
3091     return \@flags;
3092 }
3093
3094 sub config_to_command {
3095     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3096
3097     my $cmd = [];
3098     my $globalFlags = [];
3099     my $machineFlags = [];
3100     my $rtcFlags = [];
3101     my $devices = [];
3102     my $pciaddr = '';
3103     my $bridges = {};
3104     my $ostype = $conf->{ostype};
3105     my $winversion = windows_version($ostype);
3106     my $kvm = $conf->{kvm};
3107     my $nodename = nodename();
3108
3109     my $arch = get_vm_arch($conf);
3110     my $kvm_binary = get_command_for_arch($arch);
3111     my $kvmver = kvm_user_version($kvm_binary);
3112
3113     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3114         $kvmver //= "undefined";
3115         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3116     }
3117
3118     my $add_pve_version = min_version($kvmver, 4, 1);
3119
3120     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3121     my $machine_version = extract_version($machine_type, $kvmver);
3122     $kvm //= 1 if is_native($arch);
3123
3124     $machine_version =~ m/(\d+)\.(\d+)/;
3125     my ($machine_major, $machine_minor) = ($1, $2);
3126
3127     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3128         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3129     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3130         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3131             ." please upgrade node '$nodename'\n"
3132     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3133         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3134         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3135             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3136             ." node '$nodename'\n";
3137     }
3138
3139     # if a specific +pve version is required for a feature, use $version_guard
3140     # instead of min_version to allow machines to be run with the minimum
3141     # required version
3142     my $required_pve_version = 0;
3143     my $version_guard = sub {
3144         my ($major, $minor, $pve) = @_;
3145         return 0 if !min_version($machine_version, $major, $minor, $pve);
3146         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3147         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3148         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3149         return 1;
3150     };
3151
3152     if ($kvm && !defined kvm_version()) {
3153         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3154             ." or enable in BIOS.\n";
3155     }
3156
3157     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3158     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3159     my $use_old_bios_files = undef;
3160     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3161
3162     my $cpuunits = defined($conf->{cpuunits}) ?
3163             $conf->{cpuunits} : $defaults->{cpuunits};
3164
3165     push @$cmd, $kvm_binary;
3166
3167     push @$cmd, '-id', $vmid;
3168
3169     my $vmname = $conf->{name} || "vm$vmid";
3170
3171     push @$cmd, '-name', $vmname;
3172
3173     push @$cmd, '-no-shutdown';
3174
3175     my $use_virtio = 0;
3176
3177     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3178     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3179     push @$cmd, '-mon', "chardev=qmp,mode=control";
3180
3181     if (min_version($machine_version, 2, 12)) {
3182         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3183         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3184     }
3185
3186     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3187
3188     push @$cmd, '-daemonize';
3189
3190     if ($conf->{smbios1}) {
3191         my $smbios_conf = parse_smbios1($conf->{smbios1});
3192         if ($smbios_conf->{base64}) {
3193             # Do not pass base64 flag to qemu
3194             delete $smbios_conf->{base64};
3195             my $smbios_string = "";
3196             foreach my $key (keys %$smbios_conf) {
3197                 my $value;
3198                 if ($key eq "uuid") {
3199                     $value = $smbios_conf->{uuid}
3200                 } else {
3201                     $value = decode_base64($smbios_conf->{$key});
3202                 }
3203                 # qemu accepts any binary data, only commas need escaping by double comma
3204                 $value =~ s/,/,,/g;
3205                 $smbios_string .= "," . $key . "=" . $value if $value;
3206             }
3207             push @$cmd, '-smbios', "type=1" . $smbios_string;
3208         } else {
3209             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3210         }
3211     }
3212
3213     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3214         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3215         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3216
3217         my ($path, $format);
3218         if (my $efidisk = $conf->{efidisk0}) {
3219             my $d = parse_drive('efidisk0', $efidisk);
3220             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3221             $format = $d->{format};
3222             if ($storeid) {
3223                 $path = PVE::Storage::path($storecfg, $d->{file});
3224                 if (!defined($format)) {
3225                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3226                     $format = qemu_img_format($scfg, $volname);
3227                 }
3228             } else {
3229                 $path = $d->{file};
3230                 die "efidisk format must be specified\n"
3231                     if !defined($format);
3232             }
3233         } else {
3234             warn "no efidisk configured! Using temporary efivars disk.\n";
3235             $path = "/tmp/$vmid-ovmf.fd";
3236             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3237             $format = 'raw';
3238         }
3239
3240         my $size_str = "";
3241
3242         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3243             $size_str = ",size=" . (-s $ovmf_vars);
3244         }
3245
3246         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3247         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3248     }
3249
3250     # load q35 config
3251     if ($q35) {
3252         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3253         if (min_version($machine_version, 4, 0)) {
3254             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3255         } else {
3256             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3257         }
3258     }
3259
3260     if ($conf->{vmgenid}) {
3261         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3262     }
3263
3264     # add usb controllers
3265     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3266         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3267     push @$devices, @usbcontrollers if @usbcontrollers;
3268     my $vga = parse_vga($conf->{vga});
3269
3270     my $qxlnum = vga_conf_has_spice($conf->{vga});
3271     $vga->{type} = 'qxl' if $qxlnum;
3272
3273     if (!$vga->{type}) {
3274         if ($arch eq 'aarch64') {
3275             $vga->{type} = 'virtio';
3276         } elsif (min_version($machine_version, 2, 9)) {
3277             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3278         } else {
3279             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3280         }
3281     }
3282
3283     # enable absolute mouse coordinates (needed by vnc)
3284     my $tablet;
3285     if (defined($conf->{tablet})) {
3286         $tablet = $conf->{tablet};
3287     } else {
3288         $tablet = $defaults->{tablet};
3289         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3290         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3291     }
3292
3293     if ($tablet) {
3294         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3295         my $kbd = print_keyboarddevice_full($conf, $arch);
3296         push @$devices, '-device', $kbd if defined($kbd);
3297     }
3298
3299     my $bootorder = device_bootorder($conf);
3300
3301     # host pci device passthrough
3302     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3303         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3304
3305     # usb devices
3306     my $usb_dev_features = {};
3307     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3308
3309     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3310         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3311     push @$devices, @usbdevices if @usbdevices;
3312
3313     # serial devices
3314     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3315         if (my $path = $conf->{"serial$i"}) {
3316             if ($path eq 'socket') {
3317                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3318                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3319                 # On aarch64, serial0 is the UART device. Qemu only allows
3320                 # connecting UART devices via the '-serial' command line, as
3321                 # the device has a fixed slot on the hardware...
3322                 if ($arch eq 'aarch64' && $i == 0) {
3323                     push @$devices, '-serial', "chardev:serial$i";
3324                 } else {
3325                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3326                 }
3327             } else {
3328                 die "no such serial device\n" if ! -c $path;
3329                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3330                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3331             }
3332         }
3333     }
3334
3335     # parallel devices
3336     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3337         if (my $path = $conf->{"parallel$i"}) {
3338             die "no such parallel device\n" if ! -c $path;
3339             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3340             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3341             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3342         }
3343     }
3344
3345     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3346         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3347         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3348         push @$devices, @$audio_devs;
3349     }
3350
3351     my $sockets = 1;
3352     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3353     $sockets = $conf->{sockets} if  $conf->{sockets};
3354
3355     my $cores = $conf->{cores} || 1;
3356
3357     my $maxcpus = $sockets * $cores;
3358
3359     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3360
3361     my $allowed_vcpus = $cpuinfo->{cpus};
3362
3363     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3364         if ($allowed_vcpus < $maxcpus);
3365
3366     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3367
3368         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3369         for (my $i = 2; $i <= $vcpus; $i++)  {
3370             my $cpustr = print_cpu_device($conf,$i);
3371             push @$cmd, '-device', $cpustr;
3372         }
3373
3374     } else {
3375
3376         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3377     }
3378     push @$cmd, '-nodefaults';
3379
3380     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3381
3382     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3383
3384     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3385
3386     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3387         push @$devices, '-device', print_vga_device(
3388             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3389         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3390         push @$cmd,  '-vnc', "unix:$socket,password";
3391     } else {
3392         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3393         push @$cmd, '-nographic';
3394     }
3395
3396     # time drift fix
3397     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3398     my $useLocaltime = $conf->{localtime};
3399
3400     if ($winversion >= 5) { # windows
3401         $useLocaltime = 1 if !defined($conf->{localtime});
3402
3403         # use time drift fix when acpi is enabled
3404         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3405             $tdf = 1 if !defined($conf->{tdf});
3406         }
3407     }
3408
3409     if ($winversion >= 6) {
3410         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3411         push @$cmd, '-no-hpet';
3412     }
3413
3414     push @$rtcFlags, 'driftfix=slew' if $tdf;
3415
3416     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3417         push @$rtcFlags, "base=$conf->{startdate}";
3418     } elsif ($useLocaltime) {
3419         push @$rtcFlags, 'base=localtime';
3420     }
3421
3422     if ($forcecpu) {
3423         push @$cmd, '-cpu', $forcecpu;
3424     } else {
3425         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3426     }
3427
3428     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3429
3430     push @$cmd, '-S' if $conf->{freeze};
3431
3432     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3433
3434     my $guest_agent = parse_guest_agent($conf);
3435
3436     if ($guest_agent->{enabled}) {
3437         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3438         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3439
3440         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3441             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3442             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3443             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3444         } elsif ($guest_agent->{type} eq 'isa') {
3445             push @$devices, '-device', "isa-serial,chardev=qga0";
3446         }
3447     }
3448
3449     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3450     if ($rng && $version_guard->(4, 1, 2)) {
3451         check_rng_source($rng->{source});
3452
3453         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3454         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3455         my $limiter_str = "";
3456         if ($max_bytes) {
3457             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3458         }
3459
3460         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3461         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3462         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3463     }
3464
3465     my $spice_port;
3466
3467     if ($qxlnum) {
3468         if ($qxlnum > 1) {
3469             if ($winversion){
3470                 for (my $i = 1; $i < $qxlnum; $i++){
3471                     push @$devices, '-device', print_vga_device(
3472                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3473                 }
3474             } else {
3475                 # assume other OS works like Linux
3476                 my ($ram, $vram) = ("134217728", "67108864");
3477                 if ($vga->{memory}) {
3478                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3479                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3480                 }
3481                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3482                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3483             }
3484         }
3485
3486         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3487
3488         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3489         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3490         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3491
3492         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3493         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3494         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3495
3496         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3497         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3498
3499         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3500         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3501         if ($spice_enhancement->{foldersharing}) {
3502             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3503             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3504         }
3505
3506         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3507         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3508             if $spice_enhancement->{videostreaming};
3509
3510         push @$devices, '-spice', "$spice_opts";
3511     }
3512
3513     # enable balloon by default, unless explicitly disabled
3514     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3515         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3516         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3517     }
3518
3519     if ($conf->{watchdog}) {
3520         my $wdopts = parse_watchdog($conf->{watchdog});
3521         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3522         my $watchdog = $wdopts->{model} || 'i6300esb';
3523         push @$devices, '-device', "$watchdog$pciaddr";
3524         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3525     }
3526
3527     my $vollist = [];
3528     my $scsicontroller = {};
3529     my $ahcicontroller = {};
3530     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3531
3532     # Add iscsi initiator name if available
3533     if (my $initiator = get_initiator_name()) {
3534         push @$devices, '-iscsi', "initiator-name=$initiator";
3535     }
3536
3537     PVE::QemuConfig->foreach_volume($conf, sub {
3538         my ($ds, $drive) = @_;
3539
3540         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3541             push @$vollist, $drive->{file};
3542         }
3543
3544         # ignore efidisk here, already added in bios/fw handling code above
3545         return if $drive->{interface} eq 'efidisk';
3546
3547         $use_virtio = 1 if $ds =~ m/^virtio/;
3548
3549         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3550
3551         if ($drive->{interface} eq 'virtio'){
3552            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3553         }
3554
3555         if ($drive->{interface} eq 'scsi') {
3556
3557             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3558
3559             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3560                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3561
3562             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3563             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3564
3565             my $iothread = '';
3566             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3567                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3568                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3569             } elsif ($drive->{iothread}) {
3570                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3571             }
3572
3573             my $queues = '';
3574             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3575                 $queues = ",num_queues=$drive->{queues}";
3576             }
3577
3578             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3579                 if !$scsicontroller->{$controller};
3580             $scsicontroller->{$controller}=1;
3581         }
3582
3583         if ($drive->{interface} eq 'sata') {
3584             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3585             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3586             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3587                 if !$ahcicontroller->{$controller};
3588             $ahcicontroller->{$controller}=1;
3589         }
3590
3591         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3592         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3593
3594         push @$devices, '-drive',$drive_cmd;
3595         push @$devices, '-device', print_drivedevice_full(
3596             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3597     });
3598
3599     for (my $i = 0; $i < $MAX_NETS; $i++) {
3600         my $netname = "net$i";
3601
3602         next if !$conf->{$netname};
3603         my $d = parse_net($conf->{$netname});
3604         next if !$d;
3605
3606         $use_virtio = 1 if $d->{model} eq 'virtio';
3607
3608         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3609
3610         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3611         push @$devices, '-netdev', $netdevfull;
3612
3613         my $netdevicefull = print_netdevice_full(
3614             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3615
3616         push @$devices, '-device', $netdevicefull;
3617     }
3618
3619     if ($conf->{ivshmem}) {
3620         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3621
3622         my $bus;
3623         if ($q35) {
3624             $bus = print_pcie_addr("ivshmem");
3625         } else {
3626             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3627         }
3628
3629         my $ivshmem_name = $ivshmem->{name} // $vmid;
3630         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3631
3632         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3633         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3634             .",size=$ivshmem->{size}M";
3635     }
3636
3637     # pci.4 is nested in pci.1
3638     $bridges->{1} = 1 if $bridges->{4};
3639
3640     if (!$q35) {
3641         # add pci bridges
3642         if (min_version($machine_version, 2, 3)) {
3643            $bridges->{1} = 1;
3644            $bridges->{2} = 1;
3645         }
3646
3647         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3648
3649     }
3650
3651     for my $k (sort {$b cmp $a} keys %$bridges) {
3652         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3653
3654         my $k_name = $k;
3655         if ($k == 2 && $legacy_igd) {
3656             $k_name = "$k-igd";
3657         }
3658         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3659
3660         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3661         if ($q35) {
3662             # add after -readconfig pve-q35.cfg
3663             splice @$devices, 2, 0, '-device', $devstr;
3664         } else {
3665             unshift @$devices, '-device', $devstr if $k > 0;
3666         }
3667     }
3668
3669     if (!$kvm) {
3670         push @$machineFlags, 'accel=tcg';
3671     }
3672
3673     my $machine_type_min = $machine_type;
3674     if ($add_pve_version) {
3675         $machine_type_min =~ s/\+pve\d+$//;
3676         $machine_type_min .= "+pve$required_pve_version";
3677     }
3678     push @$machineFlags, "type=${machine_type_min}";
3679
3680     push @$cmd, @$devices;
3681     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3682     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3683     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3684
3685     if (my $vmstate = $conf->{vmstate}) {
3686         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3687         push @$vollist, $vmstate;
3688         push @$cmd, '-loadstate', $statepath;
3689         print "activating and using '$vmstate' as vmstate\n";
3690     }
3691
3692     # add custom args
3693     if ($conf->{args}) {
3694         my $aa = PVE::Tools::split_args($conf->{args});
3695         push @$cmd, @$aa;
3696     }
3697
3698     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3699 }
3700
3701 sub check_rng_source {
3702     my ($source) = @_;
3703
3704     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3705     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3706         if ! -e $source;
3707
3708     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3709     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3710         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3711         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3712         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3713             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3714             ." to the host.\n";
3715     }
3716 }
3717
3718 sub spice_port {
3719     my ($vmid) = @_;
3720
3721     my $res = mon_cmd($vmid, 'query-spice');
3722
3723     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3724 }
3725
3726 sub vm_devices_list {
3727     my ($vmid) = @_;
3728
3729     my $res = mon_cmd($vmid, 'query-pci');
3730     my $devices_to_check = [];
3731     my $devices = {};
3732     foreach my $pcibus (@$res) {
3733         push @$devices_to_check, @{$pcibus->{devices}},
3734     }
3735
3736     while (@$devices_to_check) {
3737         my $to_check = [];
3738         for my $d (@$devices_to_check) {
3739             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3740             next if !$d->{'pci_bridge'};
3741
3742             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3743             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3744         }
3745         $devices_to_check = $to_check;
3746     }
3747
3748     my $resblock = mon_cmd($vmid, 'query-block');
3749     foreach my $block (@$resblock) {
3750         if($block->{device} =~ m/^drive-(\S+)/){
3751                 $devices->{$1} = 1;
3752         }
3753     }
3754
3755     my $resmice = mon_cmd($vmid, 'query-mice');
3756     foreach my $mice (@$resmice) {
3757         if ($mice->{name} eq 'QEMU HID Tablet') {
3758             $devices->{tablet} = 1;
3759             last;
3760         }
3761     }
3762
3763     # for usb devices there is no query-usb
3764     # but we can iterate over the entries in
3765     # qom-list path=/machine/peripheral
3766     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3767     foreach my $per (@$resperipheral) {
3768         if ($per->{name} =~ m/^usb\d+$/) {
3769             $devices->{$per->{name}} = 1;
3770         }
3771     }
3772
3773     return $devices;
3774 }
3775
3776 sub vm_deviceplug {
3777     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3778
3779     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3780
3781     my $devices_list = vm_devices_list($vmid);
3782     return 1 if defined($devices_list->{$deviceid});
3783
3784     # add PCI bridge if we need it for the device
3785     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3786
3787     if ($deviceid eq 'tablet') {
3788
3789         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3790
3791     } elsif ($deviceid eq 'keyboard') {
3792
3793         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3794
3795     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3796
3797         die "usb hotplug currently not reliable\n";
3798         # since we can't reliably hot unplug all added usb devices and usb
3799         # passthrough breaks live migration we disable usb hotplugging for now
3800         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3801
3802     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3803
3804         qemu_iothread_add($vmid, $deviceid, $device);
3805
3806         qemu_driveadd($storecfg, $vmid, $device);
3807         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3808
3809         qemu_deviceadd($vmid, $devicefull);
3810         eval { qemu_deviceaddverify($vmid, $deviceid); };
3811         if (my $err = $@) {
3812             eval { qemu_drivedel($vmid, $deviceid); };
3813             warn $@ if $@;
3814             die $err;
3815         }
3816
3817     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3818
3819
3820         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3821         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3822         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3823
3824         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3825
3826         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3827             qemu_iothread_add($vmid, $deviceid, $device);
3828             $devicefull .= ",iothread=iothread-$deviceid";
3829         }
3830
3831         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3832             $devicefull .= ",num_queues=$device->{queues}";
3833         }
3834
3835         qemu_deviceadd($vmid, $devicefull);
3836         qemu_deviceaddverify($vmid, $deviceid);
3837
3838     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3839
3840         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3841         qemu_driveadd($storecfg, $vmid, $device);
3842
3843         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3844         eval { qemu_deviceadd($vmid, $devicefull); };
3845         if (my $err = $@) {
3846             eval { qemu_drivedel($vmid, $deviceid); };
3847             warn $@ if $@;
3848             die $err;
3849         }
3850
3851     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3852
3853         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3854
3855         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3856         my $use_old_bios_files = undef;
3857         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3858
3859         my $netdevicefull = print_netdevice_full(
3860             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3861         qemu_deviceadd($vmid, $netdevicefull);
3862         eval {
3863             qemu_deviceaddverify($vmid, $deviceid);
3864             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3865         };
3866         if (my $err = $@) {
3867             eval { qemu_netdevdel($vmid, $deviceid); };
3868             warn $@ if $@;
3869             die $err;
3870         }
3871
3872     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3873
3874         my $bridgeid = $2;
3875         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3876         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3877
3878         qemu_deviceadd($vmid, $devicefull);
3879         qemu_deviceaddverify($vmid, $deviceid);
3880
3881     } else {
3882         die "can't hotplug device '$deviceid'\n";
3883     }
3884
3885     return 1;
3886 }
3887
3888 # fixme: this should raise exceptions on error!
3889 sub vm_deviceunplug {
3890     my ($vmid, $conf, $deviceid) = @_;
3891
3892     my $devices_list = vm_devices_list($vmid);
3893     return 1 if !defined($devices_list->{$deviceid});
3894
3895     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3896     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3897
3898     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3899
3900         qemu_devicedel($vmid, $deviceid);
3901
3902     } elsif ($deviceid =~ m/^usb\d+$/) {
3903
3904         die "usb hotplug currently not reliable\n";
3905         # when unplugging usb devices this way, there may be remaining usb
3906         # controllers/hubs so we disable it for now
3907         #qemu_devicedel($vmid, $deviceid);
3908         #qemu_devicedelverify($vmid, $deviceid);
3909
3910     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3911
3912         qemu_devicedel($vmid, $deviceid);
3913         qemu_devicedelverify($vmid, $deviceid);
3914         qemu_drivedel($vmid, $deviceid);
3915         qemu_iothread_del($conf, $vmid, $deviceid);
3916
3917     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3918
3919         qemu_devicedel($vmid, $deviceid);
3920         qemu_devicedelverify($vmid, $deviceid);
3921         qemu_iothread_del($conf, $vmid, $deviceid);
3922
3923     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3924
3925         qemu_devicedel($vmid, $deviceid);
3926         qemu_drivedel($vmid, $deviceid);
3927         qemu_deletescsihw($conf, $vmid, $deviceid);
3928
3929     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3930
3931         qemu_devicedel($vmid, $deviceid);
3932         qemu_devicedelverify($vmid, $deviceid);
3933         qemu_netdevdel($vmid, $deviceid);
3934
3935     } else {
3936         die "can't unplug device '$deviceid'\n";
3937     }
3938
3939     return 1;
3940 }
3941
3942 sub qemu_deviceadd {
3943     my ($vmid, $devicefull) = @_;
3944
3945     $devicefull = "driver=".$devicefull;
3946     my %options =  split(/[=,]/, $devicefull);
3947
3948     mon_cmd($vmid, "device_add" , %options);
3949 }
3950
3951 sub qemu_devicedel {
3952     my ($vmid, $deviceid) = @_;
3953
3954     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3955 }
3956
3957 sub qemu_iothread_add {
3958     my($vmid, $deviceid, $device) = @_;
3959
3960     if ($device->{iothread}) {
3961         my $iothreads = vm_iothreads_list($vmid);
3962         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3963     }
3964 }
3965
3966 sub qemu_iothread_del {
3967     my($conf, $vmid, $deviceid) = @_;
3968
3969     my $confid = $deviceid;
3970     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3971         $confid = 'scsi' . $1;
3972     }
3973     my $device = parse_drive($confid, $conf->{$confid});
3974     if ($device->{iothread}) {
3975         my $iothreads = vm_iothreads_list($vmid);
3976         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3977     }
3978 }
3979
3980 sub qemu_objectadd {
3981     my($vmid, $objectid, $qomtype) = @_;
3982
3983     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3984
3985     return 1;
3986 }
3987
3988 sub qemu_objectdel {
3989     my($vmid, $objectid) = @_;
3990
3991     mon_cmd($vmid, "object-del", id => $objectid);
3992
3993     return 1;
3994 }
3995
3996 sub qemu_driveadd {
3997     my ($storecfg, $vmid, $device) = @_;
3998
3999     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4000     $drive =~ s/\\/\\\\/g;
4001     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4002
4003     # If the command succeeds qemu prints: "OK"
4004     return 1 if $ret =~ m/OK/s;
4005
4006     die "adding drive failed: $ret\n";
4007 }
4008
4009 sub qemu_drivedel {
4010     my($vmid, $deviceid) = @_;
4011
4012     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4013     $ret =~ s/^\s+//;
4014
4015     return 1 if $ret eq "";
4016
4017     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4018     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4019
4020     die "deleting drive $deviceid failed : $ret\n";
4021 }
4022
4023 sub qemu_deviceaddverify {
4024     my ($vmid, $deviceid) = @_;
4025
4026     for (my $i = 0; $i <= 5; $i++) {
4027          my $devices_list = vm_devices_list($vmid);
4028          return 1 if defined($devices_list->{$deviceid});
4029          sleep 1;
4030     }
4031
4032     die "error on hotplug device '$deviceid'\n";
4033 }
4034
4035
4036 sub qemu_devicedelverify {
4037     my ($vmid, $deviceid) = @_;
4038
4039     # need to verify that the device is correctly removed as device_del
4040     # is async and empty return is not reliable
4041
4042     for (my $i = 0; $i <= 5; $i++) {
4043          my $devices_list = vm_devices_list($vmid);
4044          return 1 if !defined($devices_list->{$deviceid});
4045          sleep 1;
4046     }
4047
4048     die "error on hot-unplugging device '$deviceid'\n";
4049 }
4050
4051 sub qemu_findorcreatescsihw {
4052     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4053
4054     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4055
4056     my $scsihwid="$controller_prefix$controller";
4057     my $devices_list = vm_devices_list($vmid);
4058
4059     if(!defined($devices_list->{$scsihwid})) {
4060         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4061     }
4062
4063     return 1;
4064 }
4065
4066 sub qemu_deletescsihw {
4067     my ($conf, $vmid, $opt) = @_;
4068
4069     my $device = parse_drive($opt, $conf->{$opt});
4070
4071     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4072         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4073         return 1;
4074     }
4075
4076     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4077
4078     my $devices_list = vm_devices_list($vmid);
4079     foreach my $opt (keys %{$devices_list}) {
4080         if (is_valid_drivename($opt)) {
4081             my $drive = parse_drive($opt, $conf->{$opt});
4082             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4083                 return 1;
4084             }
4085         }
4086     }
4087
4088     my $scsihwid="scsihw$controller";
4089
4090     vm_deviceunplug($vmid, $conf, $scsihwid);
4091
4092     return 1;
4093 }
4094
4095 sub qemu_add_pci_bridge {
4096     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4097
4098     my $bridges = {};
4099
4100     my $bridgeid;
4101
4102     print_pci_addr($device, $bridges, $arch, $machine_type);
4103
4104     while (my ($k, $v) = each %$bridges) {
4105         $bridgeid = $k;
4106     }
4107     return 1 if !defined($bridgeid) || $bridgeid < 1;
4108
4109     my $bridge = "pci.$bridgeid";
4110     my $devices_list = vm_devices_list($vmid);
4111
4112     if (!defined($devices_list->{$bridge})) {
4113         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4114     }
4115
4116     return 1;
4117 }
4118
4119 sub qemu_set_link_status {
4120     my ($vmid, $device, $up) = @_;
4121
4122     mon_cmd($vmid, "set_link", name => $device,
4123                up => $up ? JSON::true : JSON::false);
4124 }
4125
4126 sub qemu_netdevadd {
4127     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4128
4129     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4130     my %options =  split(/[=,]/, $netdev);
4131
4132     if (defined(my $vhost = $options{vhost})) {
4133         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4134     }
4135
4136     if (defined(my $queues = $options{queues})) {
4137         $options{queues} = $queues + 0;
4138     }
4139
4140     mon_cmd($vmid, "netdev_add",  %options);
4141     return 1;
4142 }
4143
4144 sub qemu_netdevdel {
4145     my ($vmid, $deviceid) = @_;
4146
4147     mon_cmd($vmid, "netdev_del", id => $deviceid);
4148 }
4149
4150 sub qemu_usb_hotplug {
4151     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4152
4153     return if !$device;
4154
4155     # remove the old one first
4156     vm_deviceunplug($vmid, $conf, $deviceid);
4157
4158     # check if xhci controller is necessary and available
4159     if ($device->{usb3}) {
4160
4161         my $devicelist = vm_devices_list($vmid);
4162
4163         if (!$devicelist->{xhci}) {
4164             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4165             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4166         }
4167     }
4168     my $d = parse_usb_device($device->{host});
4169     $d->{usb3} = $device->{usb3};
4170
4171     # add the new one
4172     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4173 }
4174
4175 sub qemu_cpu_hotplug {
4176     my ($vmid, $conf, $vcpus) = @_;
4177
4178     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4179
4180     my $sockets = 1;
4181     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4182     $sockets = $conf->{sockets} if  $conf->{sockets};
4183     my $cores = $conf->{cores} || 1;
4184     my $maxcpus = $sockets * $cores;
4185
4186     $vcpus = $maxcpus if !$vcpus;
4187
4188     die "you can't add more vcpus than maxcpus\n"
4189         if $vcpus > $maxcpus;
4190
4191     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4192
4193     if ($vcpus < $currentvcpus) {
4194
4195         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4196
4197             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4198                 qemu_devicedel($vmid, "cpu$i");
4199                 my $retry = 0;
4200                 my $currentrunningvcpus = undef;
4201                 while (1) {
4202                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4203                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4204                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4205                     $retry++;
4206                     sleep 1;
4207                 }
4208                 #update conf after each succesfull cpu unplug
4209                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4210                 PVE::QemuConfig->write_config($vmid, $conf);
4211             }
4212         } else {
4213             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4214         }
4215
4216         return;
4217     }
4218
4219     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4220     die "vcpus in running vm does not match its configuration\n"
4221         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4222
4223     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4224
4225         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4226             my $cpustr = print_cpu_device($conf, $i);
4227             qemu_deviceadd($vmid, $cpustr);
4228
4229             my $retry = 0;
4230             my $currentrunningvcpus = undef;
4231             while (1) {
4232                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4233                 last if scalar(@{$currentrunningvcpus}) == $i;
4234                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4235                 sleep 1;
4236                 $retry++;
4237             }
4238             #update conf after each succesfull cpu hotplug
4239             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4240             PVE::QemuConfig->write_config($vmid, $conf);
4241         }
4242     } else {
4243
4244         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4245             mon_cmd($vmid, "cpu-add", id => int($i));
4246         }
4247     }
4248 }
4249
4250 sub qemu_block_set_io_throttle {
4251     my ($vmid, $deviceid,
4252         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4253         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4254         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4255         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4256
4257     return if !check_running($vmid) ;
4258
4259     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4260         bps => int($bps),
4261         bps_rd => int($bps_rd),
4262         bps_wr => int($bps_wr),
4263         iops => int($iops),
4264         iops_rd => int($iops_rd),
4265         iops_wr => int($iops_wr),
4266         bps_max => int($bps_max),
4267         bps_rd_max => int($bps_rd_max),
4268         bps_wr_max => int($bps_wr_max),
4269         iops_max => int($iops_max),
4270         iops_rd_max => int($iops_rd_max),
4271         iops_wr_max => int($iops_wr_max),
4272         bps_max_length => int($bps_max_length),
4273         bps_rd_max_length => int($bps_rd_max_length),
4274         bps_wr_max_length => int($bps_wr_max_length),
4275         iops_max_length => int($iops_max_length),
4276         iops_rd_max_length => int($iops_rd_max_length),
4277         iops_wr_max_length => int($iops_wr_max_length),
4278     );
4279
4280 }
4281
4282 sub qemu_block_resize {
4283     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4284
4285     my $running = check_running($vmid);
4286
4287     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4288
4289     return if !$running;
4290
4291     my $padding = (1024 - $size % 1024) % 1024;
4292     $size = $size + $padding;
4293
4294     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4295
4296 }
4297
4298 sub qemu_volume_snapshot {
4299     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4300
4301     my $running = check_running($vmid);
4302
4303     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4304         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4305     } else {
4306         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4307     }
4308 }
4309
4310 sub qemu_volume_snapshot_delete {
4311     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4312
4313     my $running = check_running($vmid);
4314
4315     if($running) {
4316
4317         $running = undef;
4318         my $conf = PVE::QemuConfig->load_config($vmid);
4319         PVE::QemuConfig->foreach_volume($conf, sub {
4320             my ($ds, $drive) = @_;
4321             $running = 1 if $drive->{file} eq $volid;
4322         });
4323     }
4324
4325     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4326         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4327     } else {
4328         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4329     }
4330 }
4331
4332 sub set_migration_caps {
4333     my ($vmid, $savevm) = @_;
4334
4335     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4336
4337     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4338     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4339
4340     my $cap_ref = [];
4341
4342     my $enabled_cap = {
4343         "auto-converge" => 1,
4344         "xbzrle" => 1,
4345         "x-rdma-pin-all" => 0,
4346         "zero-blocks" => 0,
4347         "compress" => 0,
4348         "dirty-bitmaps" => $dirty_bitmaps,
4349     };
4350
4351     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4352
4353     for my $supported_capability (@$supported_capabilities) {
4354         push @$cap_ref, {
4355             capability => $supported_capability->{capability},
4356             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4357         };
4358     }
4359
4360     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4361 }
4362
4363 sub foreach_volid {
4364     my ($conf, $func, @param) = @_;
4365
4366     my $volhash = {};
4367
4368     my $test_volid = sub {
4369         my ($key, $drive, $snapname) = @_;
4370
4371         my $volid = $drive->{file};
4372         return if !$volid;
4373
4374         $volhash->{$volid}->{cdrom} //= 1;
4375         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4376
4377         my $replicate = $drive->{replicate} // 1;
4378         $volhash->{$volid}->{replicate} //= 0;
4379         $volhash->{$volid}->{replicate} = 1 if $replicate;
4380
4381         $volhash->{$volid}->{shared} //= 0;
4382         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4383
4384         $volhash->{$volid}->{referenced_in_config} //= 0;
4385         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4386
4387         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4388             if defined($snapname);
4389
4390         my $size = $drive->{size};
4391         $volhash->{$volid}->{size} //= $size if $size;
4392
4393         $volhash->{$volid}->{is_vmstate} //= 0;
4394         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4395
4396         $volhash->{$volid}->{is_unused} //= 0;
4397         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4398     };
4399
4400     my $include_opts = {
4401         extra_keys => ['vmstate'],
4402         include_unused => 1,
4403     };
4404
4405     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4406     foreach my $snapname (keys %{$conf->{snapshots}}) {
4407         my $snap = $conf->{snapshots}->{$snapname};
4408         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4409     }
4410
4411     foreach my $volid (keys %$volhash) {
4412         &$func($volid, $volhash->{$volid}, @param);
4413     }
4414 }
4415
4416 my $fast_plug_option = {
4417     'lock' => 1,
4418     'name' => 1,
4419     'onboot' => 1,
4420     'shares' => 1,
4421     'startup' => 1,
4422     'description' => 1,
4423     'protection' => 1,
4424     'vmstatestorage' => 1,
4425     'hookscript' => 1,
4426     'tags' => 1,
4427 };
4428
4429 # hotplug changes in [PENDING]
4430 # $selection hash can be used to only apply specified options, for
4431 # example: { cores => 1 } (only apply changed 'cores')
4432 # $errors ref is used to return error messages
4433 sub vmconfig_hotplug_pending {
4434     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4435
4436     my $defaults = load_defaults();
4437     my $arch = get_vm_arch($conf);
4438     my $machine_type = get_vm_machine($conf, undef, $arch);
4439
4440     # commit values which do not have any impact on running VM first
4441     # Note: those option cannot raise errors, we we do not care about
4442     # $selection and always apply them.
4443
4444     my $add_error = sub {
4445         my ($opt, $msg) = @_;
4446         $errors->{$opt} = "hotplug problem - $msg";
4447     };
4448
4449     my $changes = 0;
4450     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4451         if ($fast_plug_option->{$opt}) {
4452             $conf->{$opt} = $conf->{pending}->{$opt};
4453             delete $conf->{pending}->{$opt};
4454             $changes = 1;
4455         }
4456     }
4457
4458     if ($changes) {
4459         PVE::QemuConfig->write_config($vmid, $conf);
4460     }
4461
4462     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4463
4464     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4465     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4466     foreach my $opt (sort keys %$pending_delete_hash) {
4467         next if $selection && !$selection->{$opt};
4468         my $force = $pending_delete_hash->{$opt}->{force};
4469         eval {
4470             if ($opt eq 'hotplug') {
4471                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4472             } elsif ($opt eq 'tablet') {
4473                 die "skip\n" if !$hotplug_features->{usb};
4474                 if ($defaults->{tablet}) {
4475                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4476                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4477                         if $arch eq 'aarch64';
4478                 } else {
4479                     vm_deviceunplug($vmid, $conf, 'tablet');
4480                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4481                 }
4482             } elsif ($opt =~ m/^usb\d+/) {
4483                 die "skip\n";
4484                 # since we cannot reliably hot unplug usb devices we are disabling it
4485                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4486                 #vm_deviceunplug($vmid, $conf, $opt);
4487             } elsif ($opt eq 'vcpus') {
4488                 die "skip\n" if !$hotplug_features->{cpu};
4489                 qemu_cpu_hotplug($vmid, $conf, undef);
4490             } elsif ($opt eq 'balloon') {
4491                 # enable balloon device is not hotpluggable
4492                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4493                 # here we reset the ballooning value to memory
4494                 my $balloon = $conf->{memory} || $defaults->{memory};
4495                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4496             } elsif ($fast_plug_option->{$opt}) {
4497                 # do nothing
4498             } elsif ($opt =~ m/^net(\d+)$/) {
4499                 die "skip\n" if !$hotplug_features->{network};
4500                 vm_deviceunplug($vmid, $conf, $opt);
4501             } elsif (is_valid_drivename($opt)) {
4502                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4503                 vm_deviceunplug($vmid, $conf, $opt);
4504                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4505             } elsif ($opt =~ m/^memory$/) {
4506                 die "skip\n" if !$hotplug_features->{memory};
4507                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4508             } elsif ($opt eq 'cpuunits') {
4509                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4510             } elsif ($opt eq 'cpulimit') {
4511                 $cgroup->change_cpu_quota(-1, 100000);
4512             } else {
4513                 die "skip\n";
4514             }
4515         };
4516         if (my $err = $@) {
4517             &$add_error($opt, $err) if $err ne "skip\n";
4518         } else {
4519             delete $conf->{$opt};
4520             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4521         }
4522     }
4523
4524     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4525     $apply_pending_cloudinit = sub {
4526         return if $apply_pending_cloudinit_done; # once is enough
4527         $apply_pending_cloudinit_done = 1; # once is enough
4528
4529         my ($key, $value) = @_;
4530
4531         my @cloudinit_opts = keys %$confdesc_cloudinit;
4532         foreach my $opt (keys %{$conf->{pending}}) {
4533             next if !grep { $_ eq $opt } @cloudinit_opts;
4534             $conf->{$opt} = delete $conf->{pending}->{$opt};
4535         }
4536
4537         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4538         foreach my $opt (sort keys %$pending_delete_hash) {
4539             next if !grep { $_ eq $opt } @cloudinit_opts;
4540             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4541             delete $conf->{$opt};
4542         }
4543
4544         my $new_conf = { %$conf };
4545         $new_conf->{$key} = $value;
4546         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4547     };
4548
4549     foreach my $opt (keys %{$conf->{pending}}) {
4550         next if $selection && !$selection->{$opt};
4551         my $value = $conf->{pending}->{$opt};
4552         eval {
4553             if ($opt eq 'hotplug') {
4554                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4555             } elsif ($opt eq 'tablet') {
4556                 die "skip\n" if !$hotplug_features->{usb};
4557                 if ($value == 1) {
4558                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4559                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4560                         if $arch eq 'aarch64';
4561                 } elsif ($value == 0) {
4562                     vm_deviceunplug($vmid, $conf, 'tablet');
4563                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4564                 }
4565             } elsif ($opt =~ m/^usb\d+$/) {
4566                 die "skip\n";
4567                 # since we cannot reliably hot unplug usb devices we disable it for now
4568                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4569                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4570                 #die "skip\n" if !$d;
4571                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4572             } elsif ($opt eq 'vcpus') {
4573                 die "skip\n" if !$hotplug_features->{cpu};
4574                 qemu_cpu_hotplug($vmid, $conf, $value);
4575             } elsif ($opt eq 'balloon') {
4576                 # enable/disable balloning device is not hotpluggable
4577                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4578                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4579                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4580
4581                 # allow manual ballooning if shares is set to zero
4582                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4583                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4584                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4585                 }
4586             } elsif ($opt =~ m/^net(\d+)$/) {
4587                 # some changes can be done without hotplug
4588                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4589                                     $vmid, $opt, $value, $arch, $machine_type);
4590             } elsif (is_valid_drivename($opt)) {
4591                 die "skip\n" if $opt eq 'efidisk0';
4592                 # some changes can be done without hotplug
4593                 my $drive = parse_drive($opt, $value);
4594                 if (drive_is_cloudinit($drive)) {
4595                     &$apply_pending_cloudinit($opt, $value);
4596                 }
4597                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4598                                      $vmid, $opt, $value, $arch, $machine_type);
4599             } elsif ($opt =~ m/^memory$/) { #dimms
4600                 die "skip\n" if !$hotplug_features->{memory};
4601                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4602             } elsif ($opt eq 'cpuunits') {
4603                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4604             } elsif ($opt eq 'cpulimit') {
4605                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4606                 $cgroup->change_cpu_quota($cpulimit, 100000);
4607             } else {
4608                 die "skip\n";  # skip non-hot-pluggable options
4609             }
4610         };
4611         if (my $err = $@) {
4612             &$add_error($opt, $err) if $err ne "skip\n";
4613         } else {
4614             $conf->{$opt} = $value;
4615             delete $conf->{pending}->{$opt};
4616         }
4617     }
4618
4619     PVE::QemuConfig->write_config($vmid, $conf);
4620 }
4621
4622 sub try_deallocate_drive {
4623     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4624
4625     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4626         my $volid = $drive->{file};
4627         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4628             my $sid = PVE::Storage::parse_volume_id($volid);
4629             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4630
4631             # check if the disk is really unused
4632             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4633                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4634             PVE::Storage::vdisk_free($storecfg, $volid);
4635             return 1;
4636         } else {
4637             # If vm is not owner of this disk remove from config
4638             return 1;
4639         }
4640     }
4641
4642     return;
4643 }
4644
4645 sub vmconfig_delete_or_detach_drive {
4646     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4647
4648     my $drive = parse_drive($opt, $conf->{$opt});
4649
4650     my $rpcenv = PVE::RPCEnvironment::get();
4651     my $authuser = $rpcenv->get_user();
4652
4653     if ($force) {
4654         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4655         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4656     } else {
4657         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4658     }
4659 }
4660
4661
4662
4663 sub vmconfig_apply_pending {
4664     my ($vmid, $conf, $storecfg, $errors) = @_;
4665
4666     my $add_apply_error = sub {
4667         my ($opt, $msg) = @_;
4668         my $err_msg = "unable to apply pending change $opt : $msg";
4669         $errors->{$opt} = $err_msg;
4670         warn $err_msg;
4671     };
4672
4673     # cold plug
4674
4675     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4676     foreach my $opt (sort keys %$pending_delete_hash) {
4677         my $force = $pending_delete_hash->{$opt}->{force};
4678         eval {
4679             if ($opt =~ m/^unused/) {
4680                 die "internal error";
4681             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4682                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4683             }
4684         };
4685         if (my $err = $@) {
4686             $add_apply_error->($opt, $err);
4687         } else {
4688             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4689             delete $conf->{$opt};
4690         }
4691     }
4692
4693     PVE::QemuConfig->cleanup_pending($conf);
4694
4695     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4696         next if $opt eq 'delete'; # just to be sure
4697         eval {
4698             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4699                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4700             }
4701         };
4702         if (my $err = $@) {
4703             $add_apply_error->($opt, $err);
4704         } else {
4705             $conf->{$opt} = delete $conf->{pending}->{$opt};
4706         }
4707     }
4708
4709     # write all changes at once to avoid unnecessary i/o
4710     PVE::QemuConfig->write_config($vmid, $conf);
4711 }
4712
4713 sub vmconfig_update_net {
4714     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4715
4716     my $newnet = parse_net($value);
4717
4718     if ($conf->{$opt}) {
4719         my $oldnet = parse_net($conf->{$opt});
4720
4721         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4722             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4723             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4724             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4725
4726             # for non online change, we try to hot-unplug
4727             die "skip\n" if !$hotplug;
4728             vm_deviceunplug($vmid, $conf, $opt);
4729         } else {
4730
4731             die "internal error" if $opt !~ m/net(\d+)/;
4732             my $iface = "tap${vmid}i$1";
4733
4734             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4735                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4736                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4737                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4738                 PVE::Network::tap_unplug($iface);
4739
4740                 if ($have_sdn) {
4741                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4742                 } else {
4743                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4744                 }
4745             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4746                 # Rate can be applied on its own but any change above needs to
4747                 # include the rate in tap_plug since OVS resets everything.
4748                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4749             }
4750
4751             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4752                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4753             }
4754
4755             return 1;
4756         }
4757     }
4758
4759     if ($hotplug) {
4760         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4761     } else {
4762         die "skip\n";
4763     }
4764 }
4765
4766 sub vmconfig_update_disk {
4767     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4768
4769     my $drive = parse_drive($opt, $value);
4770
4771     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4772         my $media = $drive->{media} || 'disk';
4773         my $oldmedia = $old_drive->{media} || 'disk';
4774         die "unable to change media type\n" if $media ne $oldmedia;
4775
4776         if (!drive_is_cdrom($old_drive)) {
4777
4778             if ($drive->{file} ne $old_drive->{file}) {
4779
4780                 die "skip\n" if !$hotplug;
4781
4782                 # unplug and register as unused
4783                 vm_deviceunplug($vmid, $conf, $opt);
4784                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4785
4786             } else {
4787                 # update existing disk
4788
4789                 # skip non hotpluggable value
4790                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4791                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4792                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4793                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4794                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4795                     die "skip\n";
4796                 }
4797
4798                 # apply throttle
4799                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4800                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4801                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4802                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4803                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4804                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4805                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4806                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4807                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4808                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4809                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4810                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4811                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4812                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4813                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4814                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4815                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4816                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4817
4818                     qemu_block_set_io_throttle(
4819                         $vmid,"drive-$opt",
4820                         ($drive->{mbps} || 0)*1024*1024,
4821                         ($drive->{mbps_rd} || 0)*1024*1024,
4822                         ($drive->{mbps_wr} || 0)*1024*1024,
4823                          $drive->{iops} || 0,
4824                          $drive->{iops_rd} || 0,
4825                          $drive->{iops_wr} || 0,
4826                         ($drive->{mbps_max} || 0)*1024*1024,
4827                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4828                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4829                          $drive->{iops_max} || 0,
4830                          $drive->{iops_rd_max} || 0,
4831                          $drive->{iops_wr_max} || 0,
4832                          $drive->{bps_max_length} || 1,
4833                          $drive->{bps_rd_max_length} || 1,
4834                          $drive->{bps_wr_max_length} || 1,
4835                          $drive->{iops_max_length} || 1,
4836                          $drive->{iops_rd_max_length} || 1,
4837                          $drive->{iops_wr_max_length} || 1,
4838                     );
4839
4840                 }
4841
4842                 return 1;
4843             }
4844
4845         } else { # cdrom
4846
4847             if ($drive->{file} eq 'none') {
4848                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4849                 if (drive_is_cloudinit($old_drive)) {
4850                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4851                 }
4852             } else {
4853                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4854
4855                 # force eject if locked
4856                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4857
4858                 if ($path) {
4859                     mon_cmd($vmid, "blockdev-change-medium",
4860                         id => "$opt", filename => "$path");
4861                 }
4862             }
4863
4864             return 1;
4865         }
4866     }
4867
4868     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4869     # hotplug new disks
4870     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4871     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4872 }
4873
4874 # called in locked context by incoming migration
4875 sub vm_migrate_get_nbd_disks {
4876     my ($storecfg, $conf, $replicated_volumes) = @_;
4877
4878     my $local_volumes = {};
4879     PVE::QemuConfig->foreach_volume($conf, sub {
4880         my ($ds, $drive) = @_;
4881
4882         return if drive_is_cdrom($drive);
4883
4884         my $volid = $drive->{file};
4885
4886         return if !$volid;
4887
4888         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4889
4890         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4891         return if $scfg->{shared};
4892
4893         # replicated disks re-use existing state via bitmap
4894         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4895         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4896     });
4897     return $local_volumes;
4898 }
4899
4900 # called in locked context by incoming migration
4901 sub vm_migrate_alloc_nbd_disks {
4902     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4903
4904     my $format = undef;
4905
4906     my $nbd = {};
4907     foreach my $opt (sort keys %$source_volumes) {
4908         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4909
4910         if ($use_existing) {
4911             $nbd->{$opt}->{drivestr} = print_drive($drive);
4912             $nbd->{$opt}->{volid} = $volid;
4913             $nbd->{$opt}->{replicated} = 1;
4914             next;
4915         }
4916
4917         # If a remote storage is specified and the format of the original
4918         # volume is not available there, fall back to the default format.
4919         # Otherwise use the same format as the original.
4920         if (!$storagemap->{identity}) {
4921             $storeid = map_storage($storagemap, $storeid);
4922             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4923             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4924             my $fileFormat = qemu_img_format($scfg, $volname);
4925             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4926         } else {
4927             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4928             $format = qemu_img_format($scfg, $volname);
4929         }
4930
4931         my $size = $drive->{size} / 1024;
4932         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4933         my $newdrive = $drive;
4934         $newdrive->{format} = $format;
4935         $newdrive->{file} = $newvolid;
4936         my $drivestr = print_drive($newdrive);
4937         $nbd->{$opt}->{drivestr} = $drivestr;
4938         $nbd->{$opt}->{volid} = $newvolid;
4939     }
4940
4941     return $nbd;
4942 }
4943
4944 # see vm_start_nolock for parameters, additionally:
4945 # migrate_opts:
4946 #   storagemap = parsed storage map for allocating NBD disks
4947 sub vm_start {
4948     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4949
4950     return PVE::QemuConfig->lock_config($vmid, sub {
4951         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4952
4953         die "you can't start a vm if it's a template\n"
4954             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4955
4956         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4957         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4958
4959         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4960
4961         if ($has_backup_lock && $running) {
4962             # a backup is currently running, attempt to start the guest in the
4963             # existing QEMU instance
4964             return vm_resume($vmid);
4965         }
4966
4967         PVE::QemuConfig->check_lock($conf)
4968             if !($params->{skiplock} || $has_suspended_lock);
4969
4970         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4971
4972         die "VM $vmid already running\n" if $running;
4973
4974         if (my $storagemap = $migrate_opts->{storagemap}) {
4975             my $replicated = $migrate_opts->{replicated_volumes};
4976             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4977             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4978
4979             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4980                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4981             }
4982         }
4983
4984         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4985     });
4986 }
4987
4988
4989 # params:
4990 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4991 #   skiplock => 0/1, skip checking for config lock
4992 #   skiptemplate => 0/1, skip checking whether VM is template
4993 #   forcemachine => to force Qemu machine (rollback/migration)
4994 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4995 #   timeout => in seconds
4996 #   paused => start VM in paused state (backup)
4997 #   resume => resume from hibernation
4998 # migrate_opts:
4999 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5000 #   migratedfrom => source node
5001 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5002 #   network => CIDR of migration network
5003 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5004 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5005 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5006 sub vm_start_nolock {
5007     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5008
5009     my $statefile = $params->{statefile};
5010     my $resume = $params->{resume};
5011
5012     my $migratedfrom = $migrate_opts->{migratedfrom};
5013     my $migration_type = $migrate_opts->{type};
5014
5015     my $res = {};
5016
5017     # clean up leftover reboot request files
5018     eval { clear_reboot_request($vmid); };
5019     warn $@ if $@;
5020
5021     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5022         vmconfig_apply_pending($vmid, $conf, $storecfg);
5023         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5024     }
5025
5026     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5027
5028     my $defaults = load_defaults();
5029
5030     # set environment variable useful inside network script
5031     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5032
5033     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5034
5035     my $forcemachine = $params->{forcemachine};
5036     my $forcecpu = $params->{forcecpu};
5037     if ($resume) {
5038         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5039         $forcemachine = $conf->{runningmachine};
5040         $forcecpu = $conf->{runningcpu};
5041         print "Resuming suspended VM\n";
5042     }
5043
5044     my ($cmd, $vollist, $spice_port) =
5045         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5046
5047     my $migration_ip;
5048     my $get_migration_ip = sub {
5049         my ($nodename) = @_;
5050
5051         return $migration_ip if defined($migration_ip);
5052
5053         my $cidr = $migrate_opts->{network};
5054
5055         if (!defined($cidr)) {
5056             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5057             $cidr = $dc_conf->{migration}->{network};
5058         }
5059
5060         if (defined($cidr)) {
5061             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5062
5063             die "could not get IP: no address configured on local " .
5064                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5065
5066             die "could not get IP: multiple addresses configured on local " .
5067                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5068
5069             $migration_ip = @$ips[0];
5070         }
5071
5072         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5073             if !defined($migration_ip);
5074
5075         return $migration_ip;
5076     };
5077
5078     my $migrate_uri;
5079     if ($statefile) {
5080         if ($statefile eq 'tcp') {
5081             my $localip = "localhost";
5082             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5083             my $nodename = nodename();
5084
5085             if (!defined($migration_type)) {
5086                 if (defined($datacenterconf->{migration}->{type})) {
5087                     $migration_type = $datacenterconf->{migration}->{type};
5088                 } else {
5089                     $migration_type = 'secure';
5090                 }
5091             }
5092
5093             if ($migration_type eq 'insecure') {
5094                 $localip = $get_migration_ip->($nodename);
5095                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5096             }
5097
5098             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5099             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5100             $migrate_uri = "tcp:${localip}:${migrate_port}";
5101             push @$cmd, '-incoming', $migrate_uri;
5102             push @$cmd, '-S';
5103
5104         } elsif ($statefile eq 'unix') {
5105             # should be default for secure migrations as a ssh TCP forward
5106             # tunnel is not deterministic reliable ready and fails regurarly
5107             # to set up in time, so use UNIX socket forwards
5108             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5109             unlink $socket_addr;
5110
5111             $migrate_uri = "unix:$socket_addr";
5112
5113             push @$cmd, '-incoming', $migrate_uri;
5114             push @$cmd, '-S';
5115
5116         } elsif (-e $statefile) {
5117             push @$cmd, '-loadstate', $statefile;
5118         } else {
5119             my $statepath = PVE::Storage::path($storecfg, $statefile);
5120             push @$vollist, $statefile;
5121             push @$cmd, '-loadstate', $statepath;
5122         }
5123     } elsif ($params->{paused}) {
5124         push @$cmd, '-S';
5125     }
5126
5127     # host pci devices
5128     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5129       my $d = parse_hostpci($conf->{"hostpci$i"});
5130       next if !$d;
5131       my $pcidevices = $d->{pciid};
5132       foreach my $pcidevice (@$pcidevices) {
5133             my $pciid = $pcidevice->{id};
5134
5135             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5136             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5137             die "no pci device info for device '$pciid'\n" if !$info;
5138
5139             if ($d->{mdev}) {
5140                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5141                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5142             } else {
5143                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5144                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5145                 die "can't reset PCI device '$pciid'\n"
5146                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5147             }
5148       }
5149     }
5150
5151     PVE::Storage::activate_volumes($storecfg, $vollist);
5152
5153     eval {
5154         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5155             outfunc => sub {}, errfunc => sub {});
5156     };
5157     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5158     # timeout should be more than enough here...
5159     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5160
5161     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5162                                               : $defaults->{cpuunits};
5163
5164     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5165     my %run_params = (
5166         timeout => $statefile ? undef : $start_timeout,
5167         umask => 0077,
5168         noerr => 1,
5169     );
5170
5171     # when migrating, prefix QEMU output so other side can pick up any
5172     # errors that might occur and show the user
5173     if ($migratedfrom) {
5174         $run_params{quiet} = 1;
5175         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5176     }
5177
5178     my %properties = (
5179         Slice => 'qemu.slice',
5180         KillMode => 'none'
5181     );
5182
5183     if (PVE::CGroup::cgroup_mode() == 2) {
5184         $properties{CPUWeight} = $cpuunits;
5185     } else {
5186         $properties{CPUShares} = $cpuunits;
5187     }
5188
5189     if (my $cpulimit = $conf->{cpulimit}) {
5190         $properties{CPUQuota} = int($cpulimit * 100);
5191     }
5192     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5193
5194     my $run_qemu = sub {
5195         PVE::Tools::run_fork sub {
5196             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5197
5198             my $exitcode = run_command($cmd, %run_params);
5199             die "QEMU exited with code $exitcode\n" if $exitcode;
5200         };
5201     };
5202
5203     if ($conf->{hugepages}) {
5204
5205         my $code = sub {
5206             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5207             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5208
5209             PVE::QemuServer::Memory::hugepages_mount();
5210             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5211
5212             eval { $run_qemu->() };
5213             if (my $err = $@) {
5214                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5215                     if !$conf->{keephugepages};
5216                 die $err;
5217             }
5218
5219             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5220                 if !$conf->{keephugepages};
5221         };
5222         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5223
5224     } else {
5225         eval { $run_qemu->() };
5226     }
5227
5228     if (my $err = $@) {
5229         # deactivate volumes if start fails
5230         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5231         die "start failed: $err";
5232     }
5233
5234     print "migration listens on $migrate_uri\n" if $migrate_uri;
5235     $res->{migrate_uri} = $migrate_uri;
5236
5237     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5238         eval { mon_cmd($vmid, "cont"); };
5239         warn $@ if $@;
5240     }
5241
5242     #start nbd server for storage migration
5243     if (my $nbd = $migrate_opts->{nbd}) {
5244         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5245
5246         my $migrate_storage_uri;
5247         # nbd_protocol_version > 0 for unix socket support
5248         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5249             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5250             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5251             $migrate_storage_uri = "nbd:unix:$socket_path";
5252         } else {
5253             my $nodename = nodename();
5254             my $localip = $get_migration_ip->($nodename);
5255             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5256             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5257
5258             mon_cmd($vmid, "nbd-server-start", addr => {
5259                 type => 'inet',
5260                 data => {
5261                     host => "${localip}",
5262                     port => "${storage_migrate_port}",
5263                 },
5264             });
5265             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5266             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5267         }
5268
5269         $res->{migrate_storage_uri} = $migrate_storage_uri;
5270
5271         foreach my $opt (sort keys %$nbd) {
5272             my $drivestr = $nbd->{$opt}->{drivestr};
5273             my $volid = $nbd->{$opt}->{volid};
5274             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5275             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5276             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5277             print "re-using replicated volume: $opt - $volid\n"
5278                 if $nbd->{$opt}->{replicated};
5279
5280             $res->{drives}->{$opt} = $nbd->{$opt};
5281             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5282         }
5283     }
5284
5285     if ($migratedfrom) {
5286         eval {
5287             set_migration_caps($vmid);
5288         };
5289         warn $@ if $@;
5290
5291         if ($spice_port) {
5292             print "spice listens on port $spice_port\n";
5293             $res->{spice_port} = $spice_port;
5294             if ($migrate_opts->{spice_ticket}) {
5295                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5296                     $migrate_opts->{spice_ticket});
5297                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5298             }
5299         }
5300
5301     } else {
5302         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5303             if !$statefile && $conf->{balloon};
5304
5305         foreach my $opt (keys %$conf) {
5306             next if $opt !~  m/^net\d+$/;
5307             my $nicconf = parse_net($conf->{$opt});
5308             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5309         }
5310     }
5311
5312     mon_cmd($vmid, 'qom-set',
5313                 path => "machine/peripheral/balloon0",
5314                 property => "guest-stats-polling-interval",
5315                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5316
5317     if ($resume) {
5318         print "Resumed VM, removing state\n";
5319         if (my $vmstate = $conf->{vmstate}) {
5320             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5321             PVE::Storage::vdisk_free($storecfg, $vmstate);
5322         }
5323         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5324         PVE::QemuConfig->write_config($vmid, $conf);
5325     }
5326
5327     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5328
5329     return $res;
5330 }
5331
5332 sub vm_commandline {
5333     my ($storecfg, $vmid, $snapname) = @_;
5334
5335     my $conf = PVE::QemuConfig->load_config($vmid);
5336     my $forcemachine;
5337     my $forcecpu;
5338
5339     if ($snapname) {
5340         my $snapshot = $conf->{snapshots}->{$snapname};
5341         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5342
5343         # check for machine or CPU overrides in snapshot
5344         $forcemachine = $snapshot->{runningmachine};
5345         $forcecpu = $snapshot->{runningcpu};
5346
5347         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5348
5349         $conf = $snapshot;
5350     }
5351
5352     my $defaults = load_defaults();
5353
5354     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5355         $forcemachine, $forcecpu);
5356
5357     return PVE::Tools::cmd2string($cmd);
5358 }
5359
5360 sub vm_reset {
5361     my ($vmid, $skiplock) = @_;
5362
5363     PVE::QemuConfig->lock_config($vmid, sub {
5364
5365         my $conf = PVE::QemuConfig->load_config($vmid);
5366
5367         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5368
5369         mon_cmd($vmid, "system_reset");
5370     });
5371 }
5372
5373 sub get_vm_volumes {
5374     my ($conf) = @_;
5375
5376     my $vollist = [];
5377     foreach_volid($conf, sub {
5378         my ($volid, $attr) = @_;
5379
5380         return if $volid =~ m|^/|;
5381
5382         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5383         return if !$sid;
5384
5385         push @$vollist, $volid;
5386     });
5387
5388     return $vollist;
5389 }
5390
5391 sub vm_stop_cleanup {
5392     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5393
5394     eval {
5395
5396         if (!$keepActive) {
5397             my $vollist = get_vm_volumes($conf);
5398             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5399         }
5400
5401         foreach my $ext (qw(mon qmp pid vnc qga)) {
5402             unlink "/var/run/qemu-server/${vmid}.$ext";
5403         }
5404
5405         if ($conf->{ivshmem}) {
5406             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5407             # just delete it for now, VMs which have this already open do not
5408             # are affected, but new VMs will get a separated one. If this
5409             # becomes an issue we either add some sort of ref-counting or just
5410             # add a "don't delete on stop" flag to the ivshmem format.
5411             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5412         }
5413
5414         foreach my $key (keys %$conf) {
5415             next if $key !~ m/^hostpci(\d+)$/;
5416             my $hostpciindex = $1;
5417             my $d = parse_hostpci($conf->{$key});
5418             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5419
5420             foreach my $pci (@{$d->{pciid}}) {
5421                 my $pciid = $pci->{id};
5422                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5423             }
5424         }
5425
5426         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5427     };
5428     warn $@ if $@; # avoid errors - just warn
5429 }
5430
5431 # call only in locked context
5432 sub _do_vm_stop {
5433     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5434
5435     my $pid = check_running($vmid, $nocheck);
5436     return if !$pid;
5437
5438     my $conf;
5439     if (!$nocheck) {
5440         $conf = PVE::QemuConfig->load_config($vmid);
5441         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5442         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5443             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5444             $timeout = $opts->{down} if $opts->{down};
5445         }
5446         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5447     }
5448
5449     eval {
5450         if ($shutdown) {
5451             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5452                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5453             } else {
5454                 mon_cmd($vmid, "system_powerdown");
5455             }
5456         } else {
5457             mon_cmd($vmid, "quit");
5458         }
5459     };
5460     my $err = $@;
5461
5462     if (!$err) {
5463         $timeout = 60 if !defined($timeout);
5464
5465         my $count = 0;
5466         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5467             $count++;
5468             sleep 1;
5469         }
5470
5471         if ($count >= $timeout) {
5472             if ($force) {
5473                 warn "VM still running - terminating now with SIGTERM\n";
5474                 kill 15, $pid;
5475             } else {
5476                 die "VM quit/powerdown failed - got timeout\n";
5477             }
5478         } else {
5479             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5480             return;
5481         }
5482     } else {
5483        if (!check_running($vmid, $nocheck)) {
5484             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5485             return;
5486        }
5487        if ($force) {
5488             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5489             kill 15, $pid;
5490         } else {
5491             die "VM quit/powerdown failed\n";
5492         }
5493     }
5494
5495     # wait again
5496     $timeout = 10;
5497
5498     my $count = 0;
5499     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5500         $count++;
5501         sleep 1;
5502     }
5503
5504     if ($count >= $timeout) {
5505         warn "VM still running - terminating now with SIGKILL\n";
5506         kill 9, $pid;
5507         sleep 1;
5508     }
5509
5510     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5511 }
5512
5513 # Note: use $nocheck to skip tests if VM configuration file exists.
5514 # We need that when migration VMs to other nodes (files already moved)
5515 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5516 sub vm_stop {
5517     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5518
5519     $force = 1 if !defined($force) && !$shutdown;
5520
5521     if ($migratedfrom){
5522         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5523         kill 15, $pid if $pid;
5524         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5525         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5526         return;
5527     }
5528
5529     PVE::QemuConfig->lock_config($vmid, sub {
5530         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5531    });
5532 }
5533
5534 sub vm_reboot {
5535     my ($vmid, $timeout) = @_;
5536
5537     PVE::QemuConfig->lock_config($vmid, sub {
5538         eval {
5539
5540             # only reboot if running, as qmeventd starts it again on a stop event
5541             return if !check_running($vmid);
5542
5543             create_reboot_request($vmid);
5544
5545             my $storecfg = PVE::Storage::config();
5546             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5547
5548         };
5549         if (my $err = $@) {
5550             # avoid that the next normal shutdown will be confused for a reboot
5551             clear_reboot_request($vmid);
5552             die $err;
5553         }
5554    });
5555 }
5556
5557 # note: if using the statestorage parameter, the caller has to check privileges
5558 sub vm_suspend {
5559     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5560
5561     my $conf;
5562     my $path;
5563     my $storecfg;
5564     my $vmstate;
5565
5566     PVE::QemuConfig->lock_config($vmid, sub {
5567
5568         $conf = PVE::QemuConfig->load_config($vmid);
5569
5570         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5571         PVE::QemuConfig->check_lock($conf)
5572             if !($skiplock || $is_backing_up);
5573
5574         die "cannot suspend to disk during backup\n"
5575             if $is_backing_up && $includestate;
5576
5577         if ($includestate) {
5578             $conf->{lock} = 'suspending';
5579             my $date = strftime("%Y-%m-%d", localtime(time()));
5580             $storecfg = PVE::Storage::config();
5581             if (!$statestorage) {
5582                 $statestorage = find_vmstate_storage($conf, $storecfg);
5583                 # check permissions for the storage
5584                 my $rpcenv = PVE::RPCEnvironment::get();
5585                 if ($rpcenv->{type} ne 'cli') {
5586                     my $authuser = $rpcenv->get_user();
5587                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5588                 }
5589             }
5590
5591
5592             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5593                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5594             $path = PVE::Storage::path($storecfg, $vmstate);
5595             PVE::QemuConfig->write_config($vmid, $conf);
5596         } else {
5597             mon_cmd($vmid, "stop");
5598         }
5599     });
5600
5601     if ($includestate) {
5602         # save vm state
5603         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5604
5605         eval {
5606             set_migration_caps($vmid, 1);
5607             mon_cmd($vmid, "savevm-start", statefile => $path);
5608             for(;;) {
5609                 my $state = mon_cmd($vmid, "query-savevm");
5610                 if (!$state->{status}) {
5611                     die "savevm not active\n";
5612                 } elsif ($state->{status} eq 'active') {
5613                     sleep(1);
5614                     next;
5615                 } elsif ($state->{status} eq 'completed') {
5616                     print "State saved, quitting\n";
5617                     last;
5618                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5619                     die "query-savevm failed with error '$state->{error}'\n"
5620                 } else {
5621                     die "query-savevm returned status '$state->{status}'\n";
5622                 }
5623             }
5624         };
5625         my $err = $@;
5626
5627         PVE::QemuConfig->lock_config($vmid, sub {
5628             $conf = PVE::QemuConfig->load_config($vmid);
5629             if ($err) {
5630                 # cleanup, but leave suspending lock, to indicate something went wrong
5631                 eval {
5632                     mon_cmd($vmid, "savevm-end");
5633                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5634                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5635                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5636                     PVE::QemuConfig->write_config($vmid, $conf);
5637                 };
5638                 warn $@ if $@;
5639                 die $err;
5640             }
5641
5642             die "lock changed unexpectedly\n"
5643                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5644
5645             mon_cmd($vmid, "quit");
5646             $conf->{lock} = 'suspended';
5647             PVE::QemuConfig->write_config($vmid, $conf);
5648         });
5649     }
5650 }
5651
5652 sub vm_resume {
5653     my ($vmid, $skiplock, $nocheck) = @_;
5654
5655     PVE::QemuConfig->lock_config($vmid, sub {
5656         my $res = mon_cmd($vmid, 'query-status');
5657         my $resume_cmd = 'cont';
5658         my $reset = 0;
5659
5660         if ($res->{status}) {
5661             return if $res->{status} eq 'running'; # job done, go home
5662             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5663             $reset = 1 if $res->{status} eq 'shutdown';
5664         }
5665
5666         if (!$nocheck) {
5667
5668             my $conf = PVE::QemuConfig->load_config($vmid);
5669
5670             PVE::QemuConfig->check_lock($conf)
5671                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5672         }
5673
5674         if ($reset) {
5675             # required if a VM shuts down during a backup and we get a resume
5676             # request before the backup finishes for example
5677             mon_cmd($vmid, "system_reset");
5678         }
5679         mon_cmd($vmid, $resume_cmd);
5680     });
5681 }
5682
5683 sub vm_sendkey {
5684     my ($vmid, $skiplock, $key) = @_;
5685
5686     PVE::QemuConfig->lock_config($vmid, sub {
5687
5688         my $conf = PVE::QemuConfig->load_config($vmid);
5689
5690         # there is no qmp command, so we use the human monitor command
5691         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5692         die $res if $res ne '';
5693     });
5694 }
5695
5696 # vzdump restore implementaion
5697
5698 sub tar_archive_read_firstfile {
5699     my $archive = shift;
5700
5701     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5702
5703     # try to detect archive type first
5704     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5705         die "unable to open file '$archive'\n";
5706     my $firstfile = <$fh>;
5707     kill 15, $pid;
5708     close $fh;
5709
5710     die "ERROR: archive contaions no data\n" if !$firstfile;
5711     chomp $firstfile;
5712
5713     return $firstfile;
5714 }
5715
5716 sub tar_restore_cleanup {
5717     my ($storecfg, $statfile) = @_;
5718
5719     print STDERR "starting cleanup\n";
5720
5721     if (my $fd = IO::File->new($statfile, "r")) {
5722         while (defined(my $line = <$fd>)) {
5723             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5724                 my $volid = $2;
5725                 eval {
5726                     if ($volid =~ m|^/|) {
5727                         unlink $volid || die 'unlink failed\n';
5728                     } else {
5729                         PVE::Storage::vdisk_free($storecfg, $volid);
5730                     }
5731                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5732                 };
5733                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5734             } else {
5735                 print STDERR "unable to parse line in statfile - $line";
5736             }
5737         }
5738         $fd->close();
5739     }
5740 }
5741
5742 sub restore_file_archive {
5743     my ($archive, $vmid, $user, $opts) = @_;
5744
5745     return restore_vma_archive($archive, $vmid, $user, $opts)
5746         if $archive eq '-';
5747
5748     my $info = PVE::Storage::archive_info($archive);
5749     my $format = $opts->{format} // $info->{format};
5750     my $comp = $info->{compression};
5751
5752     # try to detect archive format
5753     if ($format eq 'tar') {
5754         return restore_tar_archive($archive, $vmid, $user, $opts);
5755     } else {
5756         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5757     }
5758 }
5759
5760 # hepler to remove disks that will not be used after restore
5761 my $restore_cleanup_oldconf = sub {
5762     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5763
5764     PVE::QemuConfig->foreach_volume($oldconf, sub {
5765         my ($ds, $drive) = @_;
5766
5767         return if drive_is_cdrom($drive, 1);
5768
5769         my $volid = $drive->{file};
5770         return if !$volid || $volid =~ m|^/|;
5771
5772         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5773         return if !$path || !$owner || ($owner != $vmid);
5774
5775         # Note: only delete disk we want to restore
5776         # other volumes will become unused
5777         if ($virtdev_hash->{$ds}) {
5778             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5779             if (my $err = $@) {
5780                 warn $err;
5781             }
5782         }
5783     });
5784
5785     # delete vmstate files, after the restore we have no snapshots anymore
5786     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5787         my $snap = $oldconf->{snapshots}->{$snapname};
5788         if ($snap->{vmstate}) {
5789             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5790             if (my $err = $@) {
5791                 warn $err;
5792             }
5793         }
5794     }
5795 };
5796
5797 # Helper to parse vzdump backup device hints
5798 #
5799 # $rpcenv: Environment, used to ckeck storage permissions
5800 # $user: User ID, to check storage permissions
5801 # $storecfg: Storage configuration
5802 # $fh: the file handle for reading the configuration
5803 # $devinfo: should contain device sizes for all backu-up'ed devices
5804 # $options: backup options (pool, default storage)
5805 #
5806 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5807 my $parse_backup_hints = sub {
5808     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5809
5810     my $virtdev_hash = {};
5811
5812     while (defined(my $line = <$fh>)) {
5813         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5814             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5815             die "archive does not contain data for drive '$virtdev'\n"
5816                 if !$devinfo->{$devname};
5817
5818             if (defined($options->{storage})) {
5819                 $storeid = $options->{storage} || 'local';
5820             } elsif (!$storeid) {
5821                 $storeid = 'local';
5822             }
5823             $format = 'raw' if !$format;
5824             $devinfo->{$devname}->{devname} = $devname;
5825             $devinfo->{$devname}->{virtdev} = $virtdev;
5826             $devinfo->{$devname}->{format} = $format;
5827             $devinfo->{$devname}->{storeid} = $storeid;
5828
5829             # check permission on storage
5830             my $pool = $options->{pool}; # todo: do we need that?
5831             if ($user ne 'root@pam') {
5832                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5833             }
5834
5835             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5836         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5837             my $virtdev = $1;
5838             my $drive = parse_drive($virtdev, $2);
5839             if (drive_is_cloudinit($drive)) {
5840                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5841                 $storeid = $options->{storage} if defined ($options->{storage});
5842                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5843                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5844
5845                 $virtdev_hash->{$virtdev} = {
5846                     format => $format,
5847                     storeid => $storeid,
5848                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5849                     is_cloudinit => 1,
5850                 };
5851             }
5852         }
5853     }
5854
5855     return $virtdev_hash;
5856 };
5857
5858 # Helper to allocate and activate all volumes required for a restore
5859 #
5860 # $storecfg: Storage configuration
5861 # $virtdev_hash: as returned by parse_backup_hints()
5862 #
5863 # Returns: { $virtdev => $volid }
5864 my $restore_allocate_devices = sub {
5865     my ($storecfg, $virtdev_hash, $vmid) = @_;
5866
5867     my $map = {};
5868     foreach my $virtdev (sort keys %$virtdev_hash) {
5869         my $d = $virtdev_hash->{$virtdev};
5870         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5871         my $storeid = $d->{storeid};
5872         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5873
5874         # test if requested format is supported
5875         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5876         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5877         $d->{format} = $defFormat if !$supported;
5878
5879         my $name;
5880         if ($d->{is_cloudinit}) {
5881             $name = "vm-$vmid-cloudinit";
5882             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5883             if ($scfg->{path}) {
5884                 $name .= ".$d->{format}";
5885             }
5886         }
5887
5888         my $volid = PVE::Storage::vdisk_alloc(
5889             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5890
5891         print STDERR "new volume ID is '$volid'\n";
5892         $d->{volid} = $volid;
5893
5894         PVE::Storage::activate_volumes($storecfg, [$volid]);
5895
5896         $map->{$virtdev} = $volid;
5897     }
5898
5899     return $map;
5900 };
5901
5902 my $restore_update_config_line = sub {
5903     my ($cookie, $vmid, $map, $line, $unique) = @_;
5904
5905     return '' if $line =~ m/^\#qmdump\#/;
5906     return '' if $line =~ m/^\#vzdump\#/;
5907     return '' if $line =~ m/^lock:/;
5908     return '' if $line =~ m/^unused\d+:/;
5909     return '' if $line =~ m/^parent:/;
5910
5911     my $res = '';
5912
5913     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5914     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5915         # try to convert old 1.X settings
5916         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5917         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5918             my ($model, $macaddr) = split(/\=/, $devconfig);
5919             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5920             my $net = {
5921                 model => $model,
5922                 bridge => "vmbr$ind",
5923                 macaddr => $macaddr,
5924             };
5925             my $netstr = print_net($net);
5926
5927             $res .= "net$cookie->{netcount}: $netstr\n";
5928             $cookie->{netcount}++;
5929         }
5930     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5931         my ($id, $netstr) = ($1, $2);
5932         my $net = parse_net($netstr);
5933         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5934         $netstr = print_net($net);
5935         $res .= "$id: $netstr\n";
5936     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5937         my $virtdev = $1;
5938         my $value = $3;
5939         my $di = parse_drive($virtdev, $value);
5940         if (defined($di->{backup}) && !$di->{backup}) {
5941             $res .= "#$line";
5942         } elsif ($map->{$virtdev}) {
5943             delete $di->{format}; # format can change on restore
5944             $di->{file} = $map->{$virtdev};
5945             $value = print_drive($di);
5946             $res .= "$virtdev: $value\n";
5947         } else {
5948             $res .= $line;
5949         }
5950     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5951         my $vmgenid = $1;
5952         if ($vmgenid ne '0') {
5953             # always generate a new vmgenid if there was a valid one setup
5954             $vmgenid = generate_uuid();
5955         }
5956         $res .= "vmgenid: $vmgenid\n";
5957     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5958         my ($uuid, $uuid_str);
5959         UUID::generate($uuid);
5960         UUID::unparse($uuid, $uuid_str);
5961         my $smbios1 = parse_smbios1($2);
5962         $smbios1->{uuid} = $uuid_str;
5963         $res .= $1.print_smbios1($smbios1)."\n";
5964     } else {
5965         $res .= $line;
5966     }
5967
5968     return $res;
5969 };
5970
5971 my $restore_deactivate_volumes = sub {
5972     my ($storecfg, $devinfo) = @_;
5973
5974     my $vollist = [];
5975     foreach my $devname (keys %$devinfo) {
5976         my $volid = $devinfo->{$devname}->{volid};
5977         push @$vollist, $volid if $volid;
5978     }
5979
5980     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5981 };
5982
5983 my $restore_destroy_volumes = sub {
5984     my ($storecfg, $devinfo) = @_;
5985
5986     foreach my $devname (keys %$devinfo) {
5987         my $volid = $devinfo->{$devname}->{volid};
5988         next if !$volid;
5989         eval {
5990             if ($volid =~ m|^/|) {
5991                 unlink $volid || die 'unlink failed\n';
5992             } else {
5993                 PVE::Storage::vdisk_free($storecfg, $volid);
5994             }
5995             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5996         };
5997         print STDERR "unable to cleanup '$volid' - $@" if $@;
5998     }
5999 };
6000
6001 sub scan_volids {
6002     my ($cfg, $vmid) = @_;
6003
6004     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6005
6006     my $volid_hash = {};
6007     foreach my $storeid (keys %$info) {
6008         foreach my $item (@{$info->{$storeid}}) {
6009             next if !($item->{volid} && $item->{size});
6010             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6011             $volid_hash->{$item->{volid}} = $item;
6012         }
6013     }
6014
6015     return $volid_hash;
6016 }
6017
6018 sub update_disk_config {
6019     my ($vmid, $conf, $volid_hash) = @_;
6020
6021     my $changes;
6022     my $prefix = "VM $vmid";
6023
6024     # used and unused disks
6025     my $referenced = {};
6026
6027     # Note: it is allowed to define multiple storages with same path (alias), so
6028     # we need to check both 'volid' and real 'path' (two different volid can point
6029     # to the same path).
6030
6031     my $referencedpath = {};
6032
6033     # update size info
6034     PVE::QemuConfig->foreach_volume($conf, sub {
6035         my ($opt, $drive) = @_;
6036
6037         my $volid = $drive->{file};
6038         return if !$volid;
6039         my $volume = $volid_hash->{$volid};
6040
6041         # mark volid as "in-use" for next step
6042         $referenced->{$volid} = 1;
6043         if ($volume && (my $path = $volume->{path})) {
6044             $referencedpath->{$path} = 1;
6045         }
6046
6047         return if drive_is_cdrom($drive);
6048         return if !$volume;
6049
6050         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6051         if (defined($updated)) {
6052             $changes = 1;
6053             $conf->{$opt} = print_drive($updated);
6054             print "$prefix ($opt): $msg\n";
6055         }
6056     });
6057
6058     # remove 'unusedX' entry if volume is used
6059     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6060         my ($opt, $drive) = @_;
6061
6062         my $volid = $drive->{file};
6063         return if !$volid;
6064
6065         my $path;
6066         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6067         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6068             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6069             $changes = 1;
6070             delete $conf->{$opt};
6071         }
6072
6073         $referenced->{$volid} = 1;
6074         $referencedpath->{$path} = 1 if $path;
6075     });
6076
6077     foreach my $volid (sort keys %$volid_hash) {
6078         next if $volid =~ m/vm-$vmid-state-/;
6079         next if $referenced->{$volid};
6080         my $path = $volid_hash->{$volid}->{path};
6081         next if !$path; # just to be sure
6082         next if $referencedpath->{$path};
6083         $changes = 1;
6084         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6085         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6086         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6087     }
6088
6089     return $changes;
6090 }
6091
6092 sub rescan {
6093     my ($vmid, $nolock, $dryrun) = @_;
6094
6095     my $cfg = PVE::Storage::config();
6096
6097     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6098     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6099     foreach my $stor (keys %{$cfg->{ids}}) {
6100         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6101     }
6102
6103     print "rescan volumes...\n";
6104     my $volid_hash = scan_volids($cfg, $vmid);
6105
6106     my $updatefn =  sub {
6107         my ($vmid) = @_;
6108
6109         my $conf = PVE::QemuConfig->load_config($vmid);
6110
6111         PVE::QemuConfig->check_lock($conf);
6112
6113         my $vm_volids = {};
6114         foreach my $volid (keys %$volid_hash) {
6115             my $info = $volid_hash->{$volid};
6116             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6117         }
6118
6119         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6120
6121         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6122     };
6123
6124     if (defined($vmid)) {
6125         if ($nolock) {
6126             &$updatefn($vmid);
6127         } else {
6128             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6129         }
6130     } else {
6131         my $vmlist = config_list();
6132         foreach my $vmid (keys %$vmlist) {
6133             if ($nolock) {
6134                 &$updatefn($vmid);
6135             } else {
6136                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6137             }
6138         }
6139     }
6140 }
6141
6142 sub restore_proxmox_backup_archive {
6143     my ($archive, $vmid, $user, $options) = @_;
6144
6145     my $storecfg = PVE::Storage::config();
6146
6147     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6148     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6149
6150     my $fingerprint = $scfg->{fingerprint};
6151     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6152
6153     my $repo = PVE::PBSClient::get_repository($scfg);
6154
6155     # This is only used for `pbs-restore`!
6156     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6157     local $ENV{PBS_PASSWORD} = $password;
6158     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6159
6160     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6161         PVE::Storage::parse_volname($storecfg, $archive);
6162
6163     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6164
6165     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6166
6167     my $tmpdir = "/var/tmp/vzdumptmp$$";
6168     rmtree $tmpdir;
6169     mkpath $tmpdir;
6170
6171     my $conffile = PVE::QemuConfig->config_file($vmid);
6172      # disable interrupts (always do cleanups)
6173     local $SIG{INT} =
6174         local $SIG{TERM} =
6175         local $SIG{QUIT} =
6176         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6177
6178     # Note: $oldconf is undef if VM does not exists
6179     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6180     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6181     my $new_conf_raw = '';
6182
6183     my $rpcenv = PVE::RPCEnvironment::get();
6184     my $devinfo = {};
6185
6186     eval {
6187         # enable interrupts
6188         local $SIG{INT} =
6189             local $SIG{TERM} =
6190             local $SIG{QUIT} =
6191             local $SIG{HUP} =
6192             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6193
6194         my $cfgfn = "$tmpdir/qemu-server.conf";
6195         my $firewall_config_fn = "$tmpdir/fw.conf";
6196         my $index_fn = "$tmpdir/index.json";
6197
6198         my $cmd = "restore";
6199
6200         my $param = [$pbs_backup_name, "index.json", $index_fn];
6201         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6202         my $index = PVE::Tools::file_get_contents($index_fn);
6203         $index = decode_json($index);
6204
6205         # print Dumper($index);
6206         foreach my $info (@{$index->{files}}) {
6207             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6208                 my $devname = $1;
6209                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6210                     $devinfo->{$devname}->{size} = $1;
6211                 } else {
6212                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6213                 }
6214             }
6215         }
6216
6217         my $is_qemu_server_backup = scalar(
6218             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6219         );
6220         if (!$is_qemu_server_backup) {
6221             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6222         }
6223         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6224
6225         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6226         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6227
6228         if ($has_firewall_config) {
6229             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6230             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6231
6232             my $pve_firewall_dir = '/etc/pve/firewall';
6233             mkdir $pve_firewall_dir; # make sure the dir exists
6234             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6235         }
6236
6237         my $fh = IO::File->new($cfgfn, "r") ||
6238             die "unable to read qemu-server.conf - $!\n";
6239
6240         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6241
6242         # fixme: rate limit?
6243
6244         # create empty/temp config
6245         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6246
6247         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6248
6249         # allocate volumes
6250         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6251
6252         foreach my $virtdev (sort keys %$virtdev_hash) {
6253             my $d = $virtdev_hash->{$virtdev};
6254             next if $d->{is_cloudinit}; # no need to restore cloudinit
6255
6256             my $volid = $d->{volid};
6257
6258             my $path = PVE::Storage::path($storecfg, $volid);
6259
6260             # This is the ONLY user of the PBS_ env vars set on top of this function!
6261             my $pbs_restore_cmd = [
6262                 '/usr/bin/pbs-restore',
6263                 '--repository', $repo,
6264                 $pbs_backup_name,
6265                 "$d->{devname}.img.fidx",
6266                 $path,
6267                 '--verbose',
6268                 ];
6269
6270             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6271             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6272
6273             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6274                 push @$pbs_restore_cmd, '--skip-zero';
6275             }
6276
6277             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6278             print "restore proxmox backup image: $dbg_cmdstring\n";
6279             run_command($pbs_restore_cmd);
6280         }
6281
6282         $fh->seek(0, 0) || die "seek failed - $!\n";
6283
6284         my $cookie = { netcount => 0 };
6285         while (defined(my $line = <$fh>)) {
6286             $new_conf_raw .= $restore_update_config_line->(
6287                 $cookie,
6288                 $vmid,
6289                 $map,
6290                 $line,
6291                 $options->{unique},
6292             );
6293         }
6294
6295         $fh->close();
6296     };
6297     my $err = $@;
6298
6299     $restore_deactivate_volumes->($storecfg, $devinfo);
6300
6301     rmtree $tmpdir;
6302
6303     if ($err) {
6304         $restore_destroy_volumes->($storecfg, $devinfo);
6305         die $err;
6306     }
6307
6308     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6309
6310     PVE::Cluster::cfs_update(); # make sure we read new file
6311
6312     eval { rescan($vmid, 1); };
6313     warn $@ if $@;
6314 }
6315
6316 sub restore_vma_archive {
6317     my ($archive, $vmid, $user, $opts, $comp) = @_;
6318
6319     my $readfrom = $archive;
6320
6321     my $cfg = PVE::Storage::config();
6322     my $commands = [];
6323     my $bwlimit = $opts->{bwlimit};
6324
6325     my $dbg_cmdstring = '';
6326     my $add_pipe = sub {
6327         my ($cmd) = @_;
6328         push @$commands, $cmd;
6329         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6330         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6331         $readfrom = '-';
6332     };
6333
6334     my $input = undef;
6335     if ($archive eq '-') {
6336         $input = '<&STDIN';
6337     } else {
6338         # If we use a backup from a PVE defined storage we also consider that
6339         # storage's rate limit:
6340         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6341         if (defined($volid)) {
6342             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6343             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6344             if ($readlimit) {
6345                 print STDERR "applying read rate limit: $readlimit\n";
6346                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6347                 $add_pipe->($cstream);
6348             }
6349         }
6350     }
6351
6352     if ($comp) {
6353         my $info = PVE::Storage::decompressor_info('vma', $comp);
6354         my $cmd = $info->{decompressor};
6355         push @$cmd, $readfrom;
6356         $add_pipe->($cmd);
6357     }
6358
6359     my $tmpdir = "/var/tmp/vzdumptmp$$";
6360     rmtree $tmpdir;
6361
6362     # disable interrupts (always do cleanups)
6363     local $SIG{INT} =
6364         local $SIG{TERM} =
6365         local $SIG{QUIT} =
6366         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6367
6368     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6369     POSIX::mkfifo($mapfifo, 0600);
6370     my $fifofh;
6371     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6372
6373     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6374
6375     my $oldtimeout;
6376     my $timeout = 5;
6377
6378     my $devinfo = {};
6379
6380     my $rpcenv = PVE::RPCEnvironment::get();
6381
6382     my $conffile = PVE::QemuConfig->config_file($vmid);
6383
6384     # Note: $oldconf is undef if VM does not exist
6385     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6386     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6387     my $new_conf_raw = '';
6388
6389     my %storage_limits;
6390
6391     my $print_devmap = sub {
6392         my $cfgfn = "$tmpdir/qemu-server.conf";
6393
6394         # we can read the config - that is already extracted
6395         my $fh = IO::File->new($cfgfn, "r") ||
6396             die "unable to read qemu-server.conf - $!\n";
6397
6398         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6399         if (-f $fwcfgfn) {
6400             my $pve_firewall_dir = '/etc/pve/firewall';
6401             mkdir $pve_firewall_dir; # make sure the dir exists
6402             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6403         }
6404
6405         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6406
6407         foreach my $info (values %{$virtdev_hash}) {
6408             my $storeid = $info->{storeid};
6409             next if defined($storage_limits{$storeid});
6410
6411             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6412             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6413             $storage_limits{$storeid} = $limit * 1024;
6414         }
6415
6416         foreach my $devname (keys %$devinfo) {
6417             die "found no device mapping information for device '$devname'\n"
6418                 if !$devinfo->{$devname}->{virtdev};
6419         }
6420
6421         # create empty/temp config
6422         if ($oldconf) {
6423             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6424             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6425         }
6426
6427         # allocate volumes
6428         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6429
6430         # print restore information to $fifofh
6431         foreach my $virtdev (sort keys %$virtdev_hash) {
6432             my $d = $virtdev_hash->{$virtdev};
6433             next if $d->{is_cloudinit}; # no need to restore cloudinit
6434
6435             my $storeid = $d->{storeid};
6436             my $volid = $d->{volid};
6437
6438             my $map_opts = '';
6439             if (my $limit = $storage_limits{$storeid}) {
6440                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6441             }
6442
6443             my $write_zeros = 1;
6444             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6445                 $write_zeros = 0;
6446             }
6447
6448             my $path = PVE::Storage::path($cfg, $volid);
6449
6450             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6451
6452             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6453         }
6454
6455         $fh->seek(0, 0) || die "seek failed - $!\n";
6456
6457         my $cookie = { netcount => 0 };
6458         while (defined(my $line = <$fh>)) {
6459             $new_conf_raw .= $restore_update_config_line->(
6460                 $cookie,
6461                 $vmid,
6462                 $map,
6463                 $line,
6464                 $opts->{unique},
6465             );
6466         }
6467
6468         $fh->close();
6469     };
6470
6471     eval {
6472         # enable interrupts
6473         local $SIG{INT} =
6474             local $SIG{TERM} =
6475             local $SIG{QUIT} =
6476             local $SIG{HUP} =
6477             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6478         local $SIG{ALRM} = sub { die "got timeout\n"; };
6479
6480         $oldtimeout = alarm($timeout);
6481
6482         my $parser = sub {
6483             my $line = shift;
6484
6485             print "$line\n";
6486
6487             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6488                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6489                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6490             } elsif ($line =~ m/^CTIME: /) {
6491                 # we correctly received the vma config, so we can disable
6492                 # the timeout now for disk allocation (set to 10 minutes, so
6493                 # that we always timeout if something goes wrong)
6494                 alarm(600);
6495                 &$print_devmap();
6496                 print $fifofh "done\n";
6497                 my $tmp = $oldtimeout || 0;
6498                 $oldtimeout = undef;
6499                 alarm($tmp);
6500                 close($fifofh);
6501                 $fifofh = undef;
6502             }
6503         };
6504
6505         print "restore vma archive: $dbg_cmdstring\n";
6506         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6507     };
6508     my $err = $@;
6509
6510     alarm($oldtimeout) if $oldtimeout;
6511
6512     $restore_deactivate_volumes->($cfg, $devinfo);
6513
6514     close($fifofh) if $fifofh;
6515     unlink $mapfifo;
6516     rmtree $tmpdir;
6517
6518     if ($err) {
6519         $restore_destroy_volumes->($cfg, $devinfo);
6520         die $err;
6521     }
6522
6523     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6524
6525     PVE::Cluster::cfs_update(); # make sure we read new file
6526
6527     eval { rescan($vmid, 1); };
6528     warn $@ if $@;
6529 }
6530
6531 sub restore_tar_archive {
6532     my ($archive, $vmid, $user, $opts) = @_;
6533
6534     if ($archive ne '-') {
6535         my $firstfile = tar_archive_read_firstfile($archive);
6536         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6537             if $firstfile ne 'qemu-server.conf';
6538     }
6539
6540     my $storecfg = PVE::Storage::config();
6541
6542     # avoid zombie disks when restoring over an existing VM -> cleanup first
6543     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6544     # skiplock=1 because qmrestore has set the 'create' lock itself already
6545     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6546     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6547
6548     my $tocmd = "/usr/lib/qemu-server/qmextract";
6549
6550     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6551     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6552     $tocmd .= ' --prealloc' if $opts->{prealloc};
6553     $tocmd .= ' --info' if $opts->{info};
6554
6555     # tar option "xf" does not autodetect compression when read from STDIN,
6556     # so we pipe to zcat
6557     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6558         PVE::Tools::shellquote("--to-command=$tocmd");
6559
6560     my $tmpdir = "/var/tmp/vzdumptmp$$";
6561     mkpath $tmpdir;
6562
6563     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6564     local $ENV{VZDUMP_VMID} = $vmid;
6565     local $ENV{VZDUMP_USER} = $user;
6566
6567     my $conffile = PVE::QemuConfig->config_file($vmid);
6568     my $new_conf_raw = '';
6569
6570     # disable interrupts (always do cleanups)
6571     local $SIG{INT} =
6572         local $SIG{TERM} =
6573         local $SIG{QUIT} =
6574         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6575
6576     eval {
6577         # enable interrupts
6578         local $SIG{INT} =
6579             local $SIG{TERM} =
6580             local $SIG{QUIT} =
6581             local $SIG{HUP} =
6582             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6583
6584         if ($archive eq '-') {
6585             print "extracting archive from STDIN\n";
6586             run_command($cmd, input => "<&STDIN");
6587         } else {
6588             print "extracting archive '$archive'\n";
6589             run_command($cmd);
6590         }
6591
6592         return if $opts->{info};
6593
6594         # read new mapping
6595         my $map = {};
6596         my $statfile = "$tmpdir/qmrestore.stat";
6597         if (my $fd = IO::File->new($statfile, "r")) {
6598             while (defined (my $line = <$fd>)) {
6599                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6600                     $map->{$1} = $2 if $1;
6601                 } else {
6602                     print STDERR "unable to parse line in statfile - $line\n";
6603                 }
6604             }
6605             $fd->close();
6606         }
6607
6608         my $confsrc = "$tmpdir/qemu-server.conf";
6609
6610         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6611
6612         my $cookie = { netcount => 0 };
6613         while (defined (my $line = <$srcfd>)) {
6614             $new_conf_raw .= $restore_update_config_line->(
6615                 $cookie,
6616                 $vmid,
6617                 $map,
6618                 $line,
6619                 $opts->{unique},
6620             );
6621         }
6622
6623         $srcfd->close();
6624     };
6625     if (my $err = $@) {
6626         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6627         die $err;
6628     }
6629
6630     rmtree $tmpdir;
6631
6632     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6633
6634     PVE::Cluster::cfs_update(); # make sure we read new file
6635
6636     eval { rescan($vmid, 1); };
6637     warn $@ if $@;
6638 };
6639
6640 sub foreach_storage_used_by_vm {
6641     my ($conf, $func) = @_;
6642
6643     my $sidhash = {};
6644
6645     PVE::QemuConfig->foreach_volume($conf, sub {
6646         my ($ds, $drive) = @_;
6647         return if drive_is_cdrom($drive);
6648
6649         my $volid = $drive->{file};
6650
6651         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6652         $sidhash->{$sid} = $sid if $sid;
6653     });
6654
6655     foreach my $sid (sort keys %$sidhash) {
6656         &$func($sid);
6657     }
6658 }
6659
6660 my $qemu_snap_storage = {
6661     rbd => 1,
6662 };
6663 sub do_snapshots_with_qemu {
6664     my ($storecfg, $volid) = @_;
6665
6666     my $storage_name = PVE::Storage::parse_volume_id($volid);
6667     my $scfg = $storecfg->{ids}->{$storage_name};
6668     die "could not find storage '$storage_name'\n" if !defined($scfg);
6669
6670     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6671         return 1;
6672     }
6673
6674     if ($volid =~ m/\.(qcow2|qed)$/){
6675         return 1;
6676     }
6677
6678     return;
6679 }
6680
6681 sub qga_check_running {
6682     my ($vmid, $nowarn) = @_;
6683
6684     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6685     if ($@) {
6686         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6687         return 0;
6688     }
6689     return 1;
6690 }
6691
6692 sub template_create {
6693     my ($vmid, $conf, $disk) = @_;
6694
6695     my $storecfg = PVE::Storage::config();
6696
6697     PVE::QemuConfig->foreach_volume($conf, sub {
6698         my ($ds, $drive) = @_;
6699
6700         return if drive_is_cdrom($drive);
6701         return if $disk && $ds ne $disk;
6702
6703         my $volid = $drive->{file};
6704         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6705
6706         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6707         $drive->{file} = $voliddst;
6708         $conf->{$ds} = print_drive($drive);
6709         PVE::QemuConfig->write_config($vmid, $conf);
6710     });
6711 }
6712
6713 sub convert_iscsi_path {
6714     my ($path) = @_;
6715
6716     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6717         my $portal = $1;
6718         my $target = $2;
6719         my $lun = $3;
6720
6721         my $initiator_name = get_initiator_name();
6722
6723         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6724                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6725     }
6726
6727     die "cannot convert iscsi path '$path', unkown format\n";
6728 }
6729
6730 sub qemu_img_convert {
6731     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6732
6733     my $storecfg = PVE::Storage::config();
6734     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6735     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6736
6737     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6738
6739     my $cachemode;
6740     my $src_path;
6741     my $src_is_iscsi = 0;
6742     my $src_format;
6743
6744     if ($src_storeid) {
6745         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6746         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6747         $src_format = qemu_img_format($src_scfg, $src_volname);
6748         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6749         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6750         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6751     } elsif (-f $src_volid) {
6752         $src_path = $src_volid;
6753         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6754             $src_format = $1;
6755         }
6756     }
6757
6758     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6759
6760     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6761     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6762     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6763     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6764
6765     my $cmd = [];
6766     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6767     push @$cmd, '-l', "snapshot.name=$snapname"
6768         if $snapname && $src_format && $src_format eq "qcow2";
6769     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6770     push @$cmd, '-T', $cachemode if defined($cachemode);
6771
6772     if ($src_is_iscsi) {
6773         push @$cmd, '--image-opts';
6774         $src_path = convert_iscsi_path($src_path);
6775     } elsif ($src_format) {
6776         push @$cmd, '-f', $src_format;
6777     }
6778
6779     if ($dst_is_iscsi) {
6780         push @$cmd, '--target-image-opts';
6781         $dst_path = convert_iscsi_path($dst_path);
6782     } else {
6783         push @$cmd, '-O', $dst_format;
6784     }
6785
6786     push @$cmd, $src_path;
6787
6788     if (!$dst_is_iscsi && $is_zero_initialized) {
6789         push @$cmd, "zeroinit:$dst_path";
6790     } else {
6791         push @$cmd, $dst_path;
6792     }
6793
6794     my $parser = sub {
6795         my $line = shift;
6796         if($line =~ m/\((\S+)\/100\%\)/){
6797             my $percent = $1;
6798             my $transferred = int($size * $percent / 100);
6799             my $remaining = $size - $transferred;
6800
6801             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6802         }
6803
6804     };
6805
6806     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6807     my $err = $@;
6808     die "copy failed: $err" if $err;
6809 }
6810
6811 sub qemu_img_format {
6812     my ($scfg, $volname) = @_;
6813
6814     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6815         return $1;
6816     } else {
6817         return "raw";
6818     }
6819 }
6820
6821 sub qemu_drive_mirror {
6822     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6823
6824     $jobs = {} if !$jobs;
6825
6826     my $qemu_target;
6827     my $format;
6828     $jobs->{"drive-$drive"} = {};
6829
6830     if ($dst_volid =~ /^nbd:/) {
6831         $qemu_target = $dst_volid;
6832         $format = "nbd";
6833     } else {
6834         my $storecfg = PVE::Storage::config();
6835         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6836
6837         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6838
6839         $format = qemu_img_format($dst_scfg, $dst_volname);
6840
6841         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6842
6843         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6844     }
6845
6846     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6847     $opts->{format} = $format if $format;
6848
6849     if (defined($src_bitmap)) {
6850         $opts->{sync} = 'incremental';
6851         $opts->{bitmap} = $src_bitmap;
6852         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6853     }
6854
6855     if (defined($bwlimit)) {
6856         $opts->{speed} = $bwlimit * 1024;
6857         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6858     } else {
6859         print "drive mirror is starting for drive-$drive\n";
6860     }
6861
6862     # if a job already runs for this device we get an error, catch it for cleanup
6863     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6864     if (my $err = $@) {
6865         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6866         warn "$@\n" if $@;
6867         die "mirroring error: $err\n";
6868     }
6869
6870     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6871 }
6872
6873 # $completion can be either
6874 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6875 # 'cancel': wait until all jobs are ready, block-job-cancel them
6876 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6877 sub qemu_drive_mirror_monitor {
6878     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6879
6880     $completion //= 'complete';
6881
6882     eval {
6883         my $err_complete = 0;
6884
6885         while (1) {
6886             die "storage migration timed out\n" if $err_complete > 300;
6887
6888             my $stats = mon_cmd($vmid, "query-block-jobs");
6889
6890             my $running_mirror_jobs = {};
6891             foreach my $stat (@$stats) {
6892                 next if $stat->{type} ne 'mirror';
6893                 $running_mirror_jobs->{$stat->{device}} = $stat;
6894             }
6895
6896             my $readycounter = 0;
6897
6898             foreach my $job (keys %$jobs) {
6899
6900                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6901                     print "$job : finished\n";
6902                     delete $jobs->{$job};
6903                     next;
6904                 }
6905
6906                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6907
6908                 my $busy = $running_mirror_jobs->{$job}->{busy};
6909                 my $ready = $running_mirror_jobs->{$job}->{ready};
6910                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6911                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6912                     my $remaining = $total - $transferred;
6913                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6914
6915                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6916                 }
6917
6918                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6919             }
6920
6921             last if scalar(keys %$jobs) == 0;
6922
6923             if ($readycounter == scalar(keys %$jobs)) {
6924                 print "all mirroring jobs are ready \n";
6925                 last if $completion eq 'skip'; #do the complete later
6926
6927                 if ($vmiddst && $vmiddst != $vmid) {
6928                     my $agent_running = $qga && qga_check_running($vmid);
6929                     if ($agent_running) {
6930                         print "freeze filesystem\n";
6931                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6932                     } else {
6933                         print "suspend vm\n";
6934                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6935                     }
6936
6937                     # if we clone a disk for a new target vm, we don't switch the disk
6938                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6939
6940                     if ($agent_running) {
6941                         print "unfreeze filesystem\n";
6942                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6943                     } else {
6944                         print "resume vm\n";
6945                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6946                     }
6947
6948                     last;
6949                 } else {
6950
6951                     foreach my $job (keys %$jobs) {
6952                         # try to switch the disk if source and destination are on the same guest
6953                         print "$job: Completing block job...\n";
6954
6955                         my $op;
6956                         if ($completion eq 'complete') {
6957                             $op = 'block-job-complete';
6958                         } elsif ($completion eq 'cancel') {
6959                             $op = 'block-job-cancel';
6960                         } else {
6961                             die "invalid completion value: $completion\n";
6962                         }
6963                         eval { mon_cmd($vmid, $op, device => $job) };
6964                         if ($@ =~ m/cannot be completed/) {
6965                             print "$job: Block job cannot be completed, try again.\n";
6966                             $err_complete++;
6967                         }else {
6968                             print "$job: Completed successfully.\n";
6969                             $jobs->{$job}->{complete} = 1;
6970                         }
6971                     }
6972                 }
6973             }
6974             sleep 1;
6975         }
6976     };
6977     my $err = $@;
6978
6979     if ($err) {
6980         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6981         die "mirroring error: $err";
6982     }
6983
6984 }
6985
6986 sub qemu_blockjobs_cancel {
6987     my ($vmid, $jobs) = @_;
6988
6989     foreach my $job (keys %$jobs) {
6990         print "$job: Cancelling block job\n";
6991         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6992         $jobs->{$job}->{cancel} = 1;
6993     }
6994
6995     while (1) {
6996         my $stats = mon_cmd($vmid, "query-block-jobs");
6997
6998         my $running_jobs = {};
6999         foreach my $stat (@$stats) {
7000             $running_jobs->{$stat->{device}} = $stat;
7001         }
7002
7003         foreach my $job (keys %$jobs) {
7004
7005             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7006                 print "$job: Done.\n";
7007                 delete $jobs->{$job};
7008             }
7009         }
7010
7011         last if scalar(keys %$jobs) == 0;
7012
7013         sleep 1;
7014     }
7015 }
7016
7017 sub clone_disk {
7018     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7019         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7020
7021     my $newvolid;
7022
7023     if (!$full) {
7024         print "create linked clone of drive $drivename ($drive->{file})\n";
7025         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7026         push @$newvollist, $newvolid;
7027     } else {
7028
7029         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7030         $storeid = $storage if $storage;
7031
7032         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7033
7034         print "create full clone of drive $drivename ($drive->{file})\n";
7035         my $name = undef;
7036         my $size = undef;
7037         if (drive_is_cloudinit($drive)) {
7038             $name = "vm-$newvmid-cloudinit";
7039             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7040             if ($scfg->{path}) {
7041                 $name .= ".$dst_format";
7042             }
7043             $snapname = undef;
7044             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7045         } elsif ($drivename eq 'efidisk0') {
7046             $size = get_efivars_size($conf);
7047         } else {
7048             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7049         }
7050         $newvolid = PVE::Storage::vdisk_alloc(
7051             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7052         );
7053         push @$newvollist, $newvolid;
7054
7055         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7056
7057         if (drive_is_cloudinit($drive)) {
7058             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7059             # if this is the case, we have to complete any block-jobs still there from
7060             # previous drive-mirrors
7061             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7062                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7063             }
7064             goto no_data_clone;
7065         }
7066
7067         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7068         if (!$running || $snapname) {
7069             # TODO: handle bwlimits
7070             if ($drivename eq 'efidisk0') {
7071                 # the relevant data on the efidisk may be smaller than the source
7072                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7073                 # that is given by the OVMF_VARS.fd
7074                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7075                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7076
7077                 # better for Ceph if block size is not too small, see bug #3324
7078                 my $bs = 1024*1024;
7079
7080                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7081                     "if=$src_path", "of=$dst_path"]);
7082             } else {
7083                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7084             }
7085         } else {
7086
7087             my $kvmver = get_running_qemu_version ($vmid);
7088             if (!min_version($kvmver, 2, 7)) {
7089                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7090                     if $drive->{iothread};
7091             }
7092
7093             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7094                 $completion, $qga, $bwlimit);
7095         }
7096     }
7097
7098 no_data_clone:
7099     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7100
7101     my $disk = $drive;
7102     $disk->{format} = undef;
7103     $disk->{file} = $newvolid;
7104     $disk->{size} = $size if defined($size);
7105
7106     return $disk;
7107 }
7108
7109 sub get_running_qemu_version {
7110     my ($vmid) = @_;
7111     my $res = mon_cmd($vmid, "query-version");
7112     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7113 }
7114
7115 sub qemu_use_old_bios_files {
7116     my ($machine_type) = @_;
7117
7118     return if !$machine_type;
7119
7120     my $use_old_bios_files = undef;
7121
7122     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7123         $machine_type = $1;
7124         $use_old_bios_files = 1;
7125     } else {
7126         my $version = extract_version($machine_type, kvm_user_version());
7127         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7128         # load new efi bios files on migration. So this hack is required to allow
7129         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7130         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7131         $use_old_bios_files = !min_version($version, 2, 4);
7132     }
7133
7134     return ($use_old_bios_files, $machine_type);
7135 }
7136
7137 sub get_efivars_size {
7138     my ($conf) = @_;
7139     my $arch = get_vm_arch($conf);
7140     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7141     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7142     return -s $ovmf_vars;
7143 }
7144
7145 sub update_efidisk_size {
7146     my ($conf) = @_;
7147
7148     return if !defined($conf->{efidisk0});
7149
7150     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7151     $disk->{size} = get_efivars_size($conf);
7152     $conf->{efidisk0} = print_drive($disk);
7153
7154     return;
7155 }
7156
7157 sub create_efidisk($$$$$) {
7158     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7159
7160     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7161     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7162
7163     my $vars_size_b = -s $ovmf_vars;
7164     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7165     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7166     PVE::Storage::activate_volumes($storecfg, [$volid]);
7167
7168     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7169     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7170
7171     return ($volid, $size/1024);
7172 }
7173
7174 sub vm_iothreads_list {
7175     my ($vmid) = @_;
7176
7177     my $res = mon_cmd($vmid, 'query-iothreads');
7178
7179     my $iothreads = {};
7180     foreach my $iothread (@$res) {
7181         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7182     }
7183
7184     return $iothreads;
7185 }
7186
7187 sub scsihw_infos {
7188     my ($conf, $drive) = @_;
7189
7190     my $maxdev = 0;
7191
7192     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7193         $maxdev = 7;
7194     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7195         $maxdev = 1;
7196     } else {
7197         $maxdev = 256;
7198     }
7199
7200     my $controller = int($drive->{index} / $maxdev);
7201     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7202         ? "virtioscsi"
7203         : "scsihw";
7204
7205     return ($maxdev, $controller, $controller_prefix);
7206 }
7207
7208 sub windows_version {
7209     my ($ostype) = @_;
7210
7211     return 0 if !$ostype;
7212
7213     my $winversion = 0;
7214
7215     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7216         $winversion = 5;
7217     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7218         $winversion = 6;
7219     } elsif ($ostype =~ m/^win(\d+)$/) {
7220         $winversion = $1;
7221     }
7222
7223     return $winversion;
7224 }
7225
7226 sub resolve_dst_disk_format {
7227         my ($storecfg, $storeid, $src_volname, $format) = @_;
7228         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7229
7230         if (!$format) {
7231             # if no target format is specified, use the source disk format as hint
7232             if ($src_volname) {
7233                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7234                 $format = qemu_img_format($scfg, $src_volname);
7235             } else {
7236                 return $defFormat;
7237             }
7238         }
7239
7240         # test if requested format is supported - else use default
7241         my $supported = grep { $_ eq $format } @$validFormats;
7242         $format = $defFormat if !$supported;
7243         return $format;
7244 }
7245
7246 # NOTE: if this logic changes, please update docs & possibly gui logic
7247 sub find_vmstate_storage {
7248     my ($conf, $storecfg) = @_;
7249
7250     # first, return storage from conf if set
7251     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7252
7253     my ($target, $shared, $local);
7254
7255     foreach_storage_used_by_vm($conf, sub {
7256         my ($sid) = @_;
7257         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7258         my $dst = $scfg->{shared} ? \$shared : \$local;
7259         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7260     });
7261
7262     # second, use shared storage where VM has at least one disk
7263     # third, use local storage where VM has at least one disk
7264     # fall back to local storage
7265     $target = $shared // $local // 'local';
7266
7267     return $target;
7268 }
7269
7270 sub generate_uuid {
7271     my ($uuid, $uuid_str);
7272     UUID::generate($uuid);
7273     UUID::unparse($uuid, $uuid_str);
7274     return $uuid_str;
7275 }
7276
7277 sub generate_smbios1_uuid {
7278     return "uuid=".generate_uuid();
7279 }
7280
7281 sub nbd_stop {
7282     my ($vmid) = @_;
7283
7284     mon_cmd($vmid, 'nbd-server-stop');
7285 }
7286
7287 sub create_reboot_request {
7288     my ($vmid) = @_;
7289     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7290         or die "failed to create reboot trigger file: $!\n";
7291     close($fh);
7292 }
7293
7294 sub clear_reboot_request {
7295     my ($vmid) = @_;
7296     my $path = "/run/qemu-server/$vmid.reboot";
7297     my $res = 0;
7298
7299     $res = unlink($path);
7300     die "could not remove reboot request for $vmid: $!"
7301         if !$res && $! != POSIX::ENOENT;
7302
7303     return $res;
7304 }
7305
7306 sub bootorder_from_legacy {
7307     my ($conf, $bootcfg) = @_;
7308
7309     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7310     my $bootindex_hash = {};
7311     my $i = 1;
7312     foreach my $o (split(//, $boot)) {
7313         $bootindex_hash->{$o} = $i*100;
7314         $i++;
7315     }
7316
7317     my $bootorder = {};
7318
7319     PVE::QemuConfig->foreach_volume($conf, sub {
7320         my ($ds, $drive) = @_;
7321
7322         if (drive_is_cdrom ($drive, 1)) {
7323             if ($bootindex_hash->{d}) {
7324                 $bootorder->{$ds} = $bootindex_hash->{d};
7325                 $bootindex_hash->{d} += 1;
7326             }
7327         } elsif ($bootindex_hash->{c}) {
7328             $bootorder->{$ds} = $bootindex_hash->{c}
7329                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7330             $bootindex_hash->{c} += 1;
7331         }
7332     });
7333
7334     if ($bootindex_hash->{n}) {
7335         for (my $i = 0; $i < $MAX_NETS; $i++) {
7336             my $netname = "net$i";
7337             next if !$conf->{$netname};
7338             $bootorder->{$netname} = $bootindex_hash->{n};
7339             $bootindex_hash->{n} += 1;
7340         }
7341     }
7342
7343     return $bootorder;
7344 }
7345
7346 # Generate default device list for 'boot: order=' property. Matches legacy
7347 # default boot order, but with explicit device names. This is important, since
7348 # the fallback for when neither 'order' nor the old format is specified relies
7349 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7350 sub get_default_bootdevices {
7351     my ($conf) = @_;
7352
7353     my @ret = ();
7354
7355     # harddisk
7356     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7357     push @ret, $first if $first;
7358
7359     # cdrom
7360     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7361     push @ret, $first if $first;
7362
7363     # network
7364     for (my $i = 0; $i < $MAX_NETS; $i++) {
7365         my $netname = "net$i";
7366         next if !$conf->{$netname};
7367         push @ret, $netname;
7368         last;
7369     }
7370
7371     return \@ret;
7372 }
7373
7374 sub device_bootorder {
7375     my ($conf) = @_;
7376
7377     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7378
7379     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7380
7381     my $bootorder = {};
7382     if (!defined($boot) || $boot->{legacy}) {
7383         $bootorder = bootorder_from_legacy($conf, $boot);
7384     } elsif ($boot->{order}) {
7385         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7386         for my $dev (PVE::Tools::split_list($boot->{order})) {
7387             $bootorder->{$dev} = $i++;
7388         }
7389     }
7390
7391     return $bootorder;
7392 }
7393
7394 # bash completion helper
7395
7396 sub complete_backup_archives {
7397     my ($cmdname, $pname, $cvalue) = @_;
7398
7399     my $cfg = PVE::Storage::config();
7400
7401     my $storeid;
7402
7403     if ($cvalue =~ m/^([^:]+):/) {
7404         $storeid = $1;
7405     }
7406
7407     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7408
7409     my $res = [];
7410     foreach my $id (keys %$data) {
7411         foreach my $item (@{$data->{$id}}) {
7412             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7413             push @$res, $item->{volid} if defined($item->{volid});
7414         }
7415     }
7416
7417     return $res;
7418 }
7419
7420 my $complete_vmid_full = sub {
7421     my ($running) = @_;
7422
7423     my $idlist = vmstatus();
7424
7425     my $res = [];
7426
7427     foreach my $id (keys %$idlist) {
7428         my $d = $idlist->{$id};
7429         if (defined($running)) {
7430             next if $d->{template};
7431             next if $running && $d->{status} ne 'running';
7432             next if !$running && $d->{status} eq 'running';
7433         }
7434         push @$res, $id;
7435
7436     }
7437     return $res;
7438 };
7439
7440 sub complete_vmid {
7441     return &$complete_vmid_full();
7442 }
7443
7444 sub complete_vmid_stopped {
7445     return &$complete_vmid_full(0);
7446 }
7447
7448 sub complete_vmid_running {
7449     return &$complete_vmid_full(1);
7450 }
7451
7452 sub complete_storage {
7453
7454     my $cfg = PVE::Storage::config();
7455     my $ids = $cfg->{ids};
7456
7457     my $res = [];
7458     foreach my $sid (keys %$ids) {
7459         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7460         next if !$ids->{$sid}->{content}->{images};
7461         push @$res, $sid;
7462     }
7463
7464     return $res;
7465 }
7466
7467 sub complete_migration_storage {
7468     my ($cmd, $param, $current_value, $all_args) = @_;
7469
7470     my $targetnode = @$all_args[1];
7471
7472     my $cfg = PVE::Storage::config();
7473     my $ids = $cfg->{ids};
7474
7475     my $res = [];
7476     foreach my $sid (keys %$ids) {
7477         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7478         next if !$ids->{$sid}->{content}->{images};
7479         push @$res, $sid;
7480     }
7481
7482     return $res;
7483 }
7484
7485 sub vm_is_paused {
7486     my ($vmid) = @_;
7487     my $qmpstatus = eval {
7488         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7489         mon_cmd($vmid, "query-status");
7490     };
7491     warn "$@\n" if $@;
7492     return $qmpstatus && $qmpstatus->{status} eq "paused";
7493 }
7494
7495 1;