PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 369     },
 370     ostype => {
 371         optional => 1,
 372         type => 'string',
 373         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 374         description => "Specify guest operating system.",
 375         verbose_description => <<EODESC,
 376 Specify guest operating system. This is used to enable special
 377 optimization/features for specific operating systems:
 378
 379 [horizontal]
 380 other;; unspecified OS
 381 wxp;; Microsoft Windows XP
 382 w2k;; Microsoft Windows 2000
 383 w2k3;; Microsoft Windows 2003
 384 w2k8;; Microsoft Windows 2008
 385 wvista;; Microsoft Windows Vista
 386 win7;; Microsoft Windows 7
 387 win8;; Microsoft Windows 8/2012/2012r2
 388 win10;; Microsoft Windows 10/2016/2019
 389 l24;; Linux 2.4 Kernel
 390 l26;; Linux 2.6 - 5.X Kernel
 391 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 392 EODESC
 393     },
 394     boot => {
 395         optional => 1,
 396         type => 'string', format => 'pve-qm-boot',
 397         description => "Specify guest boot order. Use with 'order=', usage with"
 398                      . " no key or 'legacy=' is deprecated.",
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     keephugepages => {
 440         optional => 1,
 441         type => 'boolean',
 442         default => 0,
 443         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 444             ." after VM shutdown and can be used for subsequent starts.",
 445     },
 446     vcpus => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "Number of hotplugged vcpus.",
 450         minimum => 1,
 451         default => 0,
 452     },
 453     acpi => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Enable/disable ACPI.",
 457         default => 1,
 458     },
 459     agent => {
 460         optional => 1,
 461         description => "Enable/disable Qemu GuestAgent and its properties.",
 462         type => 'string',
 463         format => $agent_fmt,
 464     },
 465     kvm => {
 466         optional => 1,
 467         type => 'boolean',
 468         description => "Enable/disable KVM hardware virtualization.",
 469         default => 1,
 470     },
 471     tdf => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable time drift fix.",
 475         default => 0,
 476     },
 477     localtime => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Set the real time clock to local time. This is enabled by default if ostype"
 481             ." indicates a Microsoft OS.",
 482     },
 483     freeze => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 487     },
 488     vga => {
 489         optional => 1,
 490         type => 'string', format => $vga_fmt,
 491         description => "Configure the VGA hardware.",
 492         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 493             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 494             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 495             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 496             ." display server. For win* OS you can select how many independent displays you want,"
 497             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 498             ." using a serial device as terminal.",
 499     },
 500     watchdog => {
 501         optional => 1,
 502         type => 'string', format => 'pve-qm-watchdog',
 503         description => "Create a virtual hardware watchdog device.",
 504         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 505             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 506             ." else the watchdog will reset the guest (or execute the respective action specified)",
 507     },
 508     startdate => {
 509         optional => 1,
 510         type => 'string',
 511         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 512         description => "Set the initial date of the real time clock. Valid format for date are:"
 513             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 514         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 515         default => 'now',
 516     },
 517     startup =>  get_standard_option('pve-startup-order'),
 518     template => {
 519         optional => 1,
 520         type => 'boolean',
 521         description => "Enable/disable Template.",
 522         default => 0,
 523     },
 524     args => {
 525         optional => 1,
 526         type => 'string',
 527         description => "Arbitrary arguments passed to kvm.",
 528         verbose_description => <<EODESCR,
 529 Arbitrary arguments passed to kvm, for example:
 530
 531 args: -no-reboot -no-hpet
 532
 533 NOTE: this option is for experts only.
 534 EODESCR
 535     },
 536     tablet => {
 537         optional => 1,
 538         type => 'boolean',
 539         default => 1,
 540         description => "Enable/disable the USB tablet device.",
 541         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 542             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 543             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 544             ." may consider disabling this to save some context switches. This is turned off by"
 545             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 546     },
 547     migrate_speed => {
 548         optional => 1,
 549         type => 'integer',
 550         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 551         minimum => 0,
 552         default => 0,
 553     },
 554     migrate_downtime => {
 555         optional => 1,
 556         type => 'number',
 557         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 558         minimum => 0,
 559         default => 0.1,
 560     },
 561     cdrom => {
 562         optional => 1,
 563         type => 'string', format => 'pve-qm-ide',
 564         typetext => '<volume>',
 565         description => "This is an alias for option -ide2",
 566     },
 567     cpu => {
 568         optional => 1,
 569         description => "Emulated CPU type.",
 570         type => 'string',
 571         format => 'pve-vm-cpu-conf',
 572     },
 573     parent => get_standard_option('pve-snapshot-name', {
 574         optional => 1,
 575         description => "Parent snapshot name. This is used internally, and should not be modified.",
 576     }),
 577     snaptime => {
 578         optional => 1,
 579         description => "Timestamp for snapshots.",
 580         type => 'integer',
 581         minimum => 0,
 582     },
 583     vmstate => {
 584         optional => 1,
 585         type => 'string', format => 'pve-volume-id',
 586         description => "Reference to a volume which stores the VM state. This is used internally"
 587             ." for snapshots.",
 588     },
 589     vmstatestorage => get_standard_option('pve-storage-id', {
 590         description => "Default storage for VM state volumes/files.",
 591         optional => 1,
 592     }),
 593     runningmachine => get_standard_option('pve-qemu-machine', {
 594         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 595             ." for snapshots.",
 596     }),
 597     runningcpu => {
 598         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 599             ." internally for snapshots.",
 600         optional => 1,
 601         type => 'string',
 602         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 603         format_description => 'QEMU -cpu parameter'
 604     },
 605     machine => get_standard_option('pve-qemu-machine'),
 606     arch => {
 607         description => "Virtual processor architecture. Defaults to the host.",
 608         optional => 1,
 609         type => 'string',
 610         enum => [qw(x86_64 aarch64)],
 611     },
 612     smbios1 => {
 613         description => "Specify SMBIOS type 1 fields.",
 614         type => 'string', format => 'pve-qm-smbios1',
 615         maxLength => 512,
 616         optional => 1,
 617     },
 618     protection => {
 619         optional => 1,
 620         type => 'boolean',
 621         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 622             ." remove disk operations.",
 623         default => 0,
 624     },
 625     bios => {
 626         optional => 1,
 627         type => 'string',
 628         enum => [ qw(seabios ovmf) ],
 629         description => "Select BIOS implementation.",
 630         default => 'seabios',
 631     },
 632     vmgenid => {
 633         type => 'string',
 634         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 635         format_description => 'UUID',
 636         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 637             ." to disable explicitly.",
 638         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 639             ." value identifier to the guest OS. This allows to notify the guest operating system"
 640             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 641             ." execution or creation from a template). The guest operating system notices the"
 642             ." change, and is then able to react as appropriate by marking its copies of"
 643             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 644             ."Note that auto-creation only works when done through API/CLI create or update methods"
 645             .", but not when manually editing the config file.",
 646         default => "1 (autogenerated)",
 647         optional => 1,
 648     },
 649     hookscript => {
 650         type => 'string',
 651         format => 'pve-volume-id',
 652         optional => 1,
 653         description => "Script that will be executed during various steps in the vms lifetime.",
 654     },
 655     ivshmem => {
 656         type => 'string',
 657         format => $ivshmem_fmt,
 658         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 659             ." the host.",
 660         optional => 1,
 661     },
 662     audio0 => {
 663         type => 'string',
 664         format => $audio_fmt,
 665         description => "Configure a audio device, useful in combination with QXL/Spice.",
 666         optional => 1
 667     },
 668     spice_enhancements => {
 669         type => 'string',
 670         format => $spice_enhancements_fmt,
 671         description => "Configure additional enhancements for SPICE.",
 672         optional => 1
 673     },
 674     tags => {
 675         type => 'string', format => 'pve-tag-list',
 676         description => 'Tags of the VM. This is only meta information.',
 677         optional => 1,
 678     },
 679     rng0 => {
 680         type => 'string',
 681         format => $rng_fmt,
 682         description => "Configure a VirtIO-based Random Number Generator.",
 683         optional => 1,
 684     },
 685 };
 686
 687 my $cicustom_fmt = {
 688     meta => {
 689         type => 'string',
 690         optional => 1,
 691         description => 'Specify a custom file containing all meta data passed to the VM via"
 692             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 693         format => 'pve-volume-id',
 694         format_description => 'volume',
 695     },
 696     network => {
 697         type => 'string',
 698         optional => 1,
 699         description => 'Specify a custom file containing all network data passed to the VM via'
 700             .' cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704     user => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all user data passed to the VM via'
 708             .' cloud-init.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712 };
 713 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 714
 715 my $confdesc_cloudinit = {
 716     citype => {
 717         optional => 1,
 718         type => 'string',
 719         description => 'Specifies the cloud-init configuration format. The default depends on the'
 720             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 721             .' and `configdrive2` for windows.',
 722         enum => ['configdrive2', 'nocloud', 'opennebula'],
 723     },
 724     ciuser => {
 725         optional => 1,
 726         type => 'string',
 727         description => "cloud-init: User name to change ssh keys and password for instead of the"
 728             ." image's configured default user.",
 729     },
 730     cipassword => {
 731         optional => 1,
 732         type => 'string',
 733         description => 'cloud-init: Password to assign the user. Using this is generally not'
 734             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 735             .' support hashed passwords.',
 736     },
 737     cicustom => {
 738         optional => 1,
 739         type => 'string',
 740         description => 'cloud-init: Specify custom files to replace the automatically generated'
 741             .' ones at start.',
 742         format => 'pve-qm-cicustom',
 743     },
 744     searchdomain => {
 745         optional => 1,
 746         type => 'string',
 747         description => "cloud-init: Sets DNS search domains for a container. Create will'
 748             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 749             .' are set.",
 750     },
 751     nameserver => {
 752         optional => 1,
 753         type => 'string', format => 'address-list',
 754         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 755             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 756             .' are set.",
 757     },
 758     sshkeys => {
 759         optional => 1,
 760         type => 'string',
 761         format => 'urlencoded',
 762         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 763     },
 764 };
 765
 766 # what about other qemu settings ?
 767 #cpu => 'string',
 768 #machine => 'string',
 769 #fda => 'file',
 770 #fdb => 'file',
 771 #mtdblock => 'file',
 772 #sd => 'file',
 773 #pflash => 'file',
 774 #snapshot => 'bool',
 775 #bootp => 'file',
 776 ##tftp => 'dir',
 777 ##smb => 'dir',
 778 #kernel => 'file',
 779 #append => 'string',
 780 #initrd => 'file',
 781 ##soundhw => 'string',
 782
 783 while (my ($k, $v) = each %$confdesc) {
 784     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 785 }
 786
 787 my $MAX_USB_DEVICES = 5;
 788 my $MAX_NETS = 32;
 789 my $MAX_SERIAL_PORTS = 4;
 790 my $MAX_PARALLEL_PORTS = 3;
 791 my $MAX_NUMA = 8;
 792
 793 my $numa_fmt = {
 794     cpus => {
 795         type => "string",
 796         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 797         description => "CPUs accessing this NUMA node.",
 798         format_description => "id[-id];...",
 799     },
 800     memory => {
 801         type => "number",
 802         description => "Amount of memory this NUMA node provides.",
 803         optional => 1,
 804     },
 805     hostnodes => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "Host NUMA nodes to use.",
 809         format_description => "id[-id];...",
 810         optional => 1,
 811     },
 812     policy => {
 813         type => 'string',
 814         enum => [qw(preferred bind interleave)],
 815         description => "NUMA allocation policy.",
 816         optional => 1,
 817     },
 818 };
 819 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 820 my $numadesc = {
 821     optional => 1,
 822     type => 'string', format => $numa_fmt,
 823     description => "NUMA topology.",
 824 };
 825 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 826
 827 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 828     $confdesc->{"numa$i"} = $numadesc;
 829 }
 830
 831 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 832                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 833                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 834 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 835
 836 my $net_fmt_bridge_descr = <<__EOD__;
 837 Bridge to attach the network device to. The Proxmox VE standard bridge
 838 is called 'vmbr0'.
 839
 840 If you do not specify a bridge, we create a kvm user (NATed) network
 841 device, which provides DHCP and DNS services. The following addresses
 842 are used:
 843
 844  10.0.2.2   Gateway
 845  10.0.2.3   DNS Server
 846  10.0.2.4   SMB Server
 847
 848 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 849 __EOD__
 850
 851 my $net_fmt = {
 852     macaddr  => get_standard_option('mac-addr', {
 853         description => "MAC address. That address must be unique withing your network. This is"
 854             ." automatically generated if not specified.",
 855     }),
 856     model => {
 857         type => 'string',
 858         description => "Network Card Model. The 'virtio' model provides the best performance with"
 859             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 860             ." best to use 'e1000'.",
 861         enum => $nic_model_list,
 862         default_key => 1,
 863     },
 864     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 865     bridge => {
 866         type => 'string',
 867         description => $net_fmt_bridge_descr,
 868         format_description => 'bridge',
 869         pattern => '[-_.\w\d]+',
 870         optional => 1,
 871     },
 872     queues => {
 873         type => 'integer',
 874         minimum => 0, maximum => 16,
 875         description => 'Number of packet queues to be used on the device.',
 876         optional => 1,
 877     },
 878     rate => {
 879         type => 'number',
 880         minimum => 0,
 881         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 882         optional => 1,
 883     },
 884     tag => {
 885         type => 'integer',
 886         minimum => 1, maximum => 4094,
 887         description => 'VLAN tag to apply to packets on this interface.',
 888         optional => 1,
 889     },
 890     trunks => {
 891         type => 'string',
 892         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 893         description => 'VLAN trunks to pass through this interface.',
 894         format_description => 'vlanid[;vlanid...]',
 895         optional => 1,
 896     },
 897     firewall => {
 898         type => 'boolean',
 899         description => 'Whether this interface should be protected by the firewall.',
 900         optional => 1,
 901     },
 902     link_down => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be disconnected (like pulling the plug).',
 905         optional => 1,
 906     },
 907     mtu => {
 908         type => 'integer',
 909         minimum => 1, maximum => 65520,
 910         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 911         optional => 1,
 912     },
 913 };
 914
 915 my $netdesc = {
 916     optional => 1,
 917     type => 'string', format => $net_fmt,
 918     description => "Specify network devices.",
 919 };
 920
 921 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 922
 923 my $ipconfig_fmt = {
 924     ip => {
 925         type => 'string',
 926         format => 'pve-ipv4-config',
 927         format_description => 'IPv4Format/CIDR',
 928         description => 'IPv4 address in CIDR format.',
 929         optional => 1,
 930         default => 'dhcp',
 931     },
 932     gw => {
 933         type => 'string',
 934         format => 'ipv4',
 935         format_description => 'GatewayIPv4',
 936         description => 'Default gateway for IPv4 traffic.',
 937         optional => 1,
 938         requires => 'ip',
 939     },
 940     ip6 => {
 941         type => 'string',
 942         format => 'pve-ipv6-config',
 943         format_description => 'IPv6Format/CIDR',
 944         description => 'IPv6 address in CIDR format.',
 945         optional => 1,
 946         default => 'dhcp',
 947     },
 948     gw6 => {
 949         type => 'string',
 950         format => 'ipv6',
 951         format_description => 'GatewayIPv6',
 952         description => 'Default gateway for IPv6 traffic.',
 953         optional => 1,
 954         requires => 'ip6',
 955     },
 956 };
 957 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 958 my $ipconfigdesc = {
 959     optional => 1,
 960     type => 'string', format => 'pve-qm-ipconfig',
 961     description => <<'EODESCR',
 962 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 963
 964 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 965
 966 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 967 gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 969 cloud-init 19.4 or newer.
 970
 971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 972 dhcp on IPv4.
 973 EODESCR
 974 };
 975 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 976
 977 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 978     $confdesc->{"net$i"} = $netdesc;
 979     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 980 }
 981
 982 foreach my $key (keys %$confdesc_cloudinit) {
 983     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 984 }
 985
 986 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 987 sub verify_volume_id_or_qm_path {
 988     my ($volid, $noerr) = @_;
 989
 990     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 991         return $volid;
 992     }
 993
 994     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 995     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 996     if ($@) {
 997         return if $noerr;
 998         die $@;
 999     }
1000     return $volid;
1001 }
1002
1003 my $usb_fmt = {
1004     host => {
1005         default_key => 1,
1006         type => 'string', format => 'pve-qm-usb-device',
1007         format_description => 'HOSTUSBDEVICE|spice',
1008         description => <<EODESCR,
1009 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1010
1011  'bus-port(.port)*' (decimal numbers) or
1012  'vendor_id:product_id' (hexadeciaml numbers) or
1013  'spice'
1014
1015 You can use the 'lsusb -t' command to list existing usb devices.
1016
1017 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1018 machines - use with special care.
1019
1020 The value 'spice' can be used to add a usb redirection devices for spice.
1021 EODESCR
1022     },
1023     usb3 => {
1024         optional => 1,
1025         type => 'boolean',
1026         description => "Specifies whether if given host option is a USB3 device or port.",
1027         default => 0,
1028     },
1029 };
1030
1031 my $usbdesc = {
1032     optional => 1,
1033     type => 'string', format => $usb_fmt,
1034     description => "Configure an USB device (n is 0 to 4).",
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1037
1038 my $serialdesc = {
1039         optional => 1,
1040         type => 'string',
1041         pattern => '(/dev/.+|socket)',
1042         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1043         verbose_description =>  <<EODESCR,
1044 Create a serial device inside the VM (n is 0 to 3), and pass through a
1045 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046 host side (use 'qm terminal' to open a terminal connection).
1047
1048 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1049 use with special care.
1050
1051 CAUTION: Experimental! User reported problems with this option.
1052 EODESCR
1053 };
1054
1055 my $paralleldesc= {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1059         description =>  "Map host parallel devices (n is 0 to 2).",
1060         verbose_description =>  <<EODESCR,
1061 Map host parallel devices (n is 0 to 2).
1062
1063 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1064 machines - use with special care.
1065
1066 CAUTION: Experimental! User reported problems with this option.
1067 EODESCR
1068 };
1069
1070 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1071     $confdesc->{"parallel$i"} = $paralleldesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1075     $confdesc->{"serial$i"} = $serialdesc;
1076 }
1077
1078 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1079     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1080 }
1081
1082 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1083     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1084 }
1085
1086 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1087     $confdesc->{"usb$i"} = $usbdesc;
1088 }
1089
1090 my $boot_fmt = {
1091     legacy => {
1092         optional => 1,
1093         default_key => 1,
1094         type => 'string',
1095         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1096                      . " Deprecated, use 'order=' instead.",
1097         pattern => '[acdn]{1,4}',
1098         format_description => "[acdn]{1,4}",
1099
1100         # note: this is also the fallback if boot: is not given at all
1101         default => 'cdn',
1102     },
1103     order => {
1104         optional => 1,
1105         type => 'string',
1106         format => 'pve-qm-bootdev-list',
1107         format_description => "device[;device...]",
1108         description => <<EODESC,
1109 The guest will attempt to boot from devices in the order they appear here.
1110
1111 Disks, optical drives and passed-through storage USB devices will be directly
1112 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1113 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1114
1115 Note that only devices in this list will be marked as bootable and thus loaded
1116 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1117 (e.g. software-raid), you need to specify all of them here.
1118
1119 Overrides the deprecated 'legacy=[acdn]*' value when given.
1120 EODESC
1121     },
1122 };
1123 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1124
1125 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1126 sub verify_bootdev {
1127     my ($dev, $noerr) = @_;
1128
1129     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1130
1131     my $check = sub {
1132         my ($base) = @_;
1133         return 0 if $dev !~ m/^$base\d+$/;
1134         return 0 if !$confdesc->{$dev};
1135         return 1;
1136     };
1137
1138     return $dev if $check->("net");
1139     return $dev if $check->("usb");
1140     return $dev if $check->("hostpci");
1141
1142     return if $noerr;
1143     die "invalid boot device '$dev'\n";
1144 }
1145
1146 sub print_bootorder {
1147     my ($devs) = @_;
1148     return "" if !@$devs;
1149     my $data = { order => join(';', @$devs) };
1150     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1151 }
1152
1153 my $kvm_api_version = 0;
1154
1155 sub kvm_version {
1156     return $kvm_api_version if $kvm_api_version;
1157
1158     open my $fh, '<', '/dev/kvm' or return;
1159
1160     # 0xae00 => KVM_GET_API_VERSION
1161     $kvm_api_version = ioctl($fh, 0xae00, 0);
1162     close($fh);
1163
1164     return $kvm_api_version;
1165 }
1166
1167 my $kvm_user_version = {};
1168 my $kvm_mtime = {};
1169
1170 sub kvm_user_version {
1171     my ($binary) = @_;
1172
1173     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1174     my $st = stat($binary);
1175
1176     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1177     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1178         $cachedmtime == $st->mtime;
1179
1180     $kvm_user_version->{$binary} = 'unknown';
1181     $kvm_mtime->{$binary} = $st->mtime;
1182
1183     my $code = sub {
1184         my $line = shift;
1185         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186             $kvm_user_version->{$binary} = $2;
1187         }
1188     };
1189
1190     eval { run_command([$binary, '--version'], outfunc => $code); };
1191     warn $@ if $@;
1192
1193     return $kvm_user_version->{$binary};
1194
1195 }
1196 my sub extract_version {
1197     my ($machine_type, $version) = @_;
1198     $version = kvm_user_version() if !defined($version);
1199     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1200 }
1201
1202 sub kernel_has_vhost_net {
1203     return -c '/dev/vhost-net';
1204 }
1205
1206 sub option_exists {
1207     my $key = shift;
1208     return defined($confdesc->{$key});
1209 }
1210
1211 my $cdrom_path;
1212 sub get_cdrom_path {
1213
1214     return  $cdrom_path if $cdrom_path;
1215
1216     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1217     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1218     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1219 }
1220
1221 sub get_iso_path {
1222     my ($storecfg, $vmid, $cdrom) = @_;
1223
1224     if ($cdrom eq 'cdrom') {
1225         return get_cdrom_path();
1226     } elsif ($cdrom eq 'none') {
1227         return '';
1228     } elsif ($cdrom =~ m|^/|) {
1229         return $cdrom;
1230     } else {
1231         return PVE::Storage::path($storecfg, $cdrom);
1232     }
1233 }
1234
1235 # try to convert old style file names to volume IDs
1236 sub filename_to_volume_id {
1237     my ($vmid, $file, $media) = @_;
1238
1239      if (!($file eq 'none' || $file eq 'cdrom' ||
1240           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1241
1242         return if $file =~ m|/|;
1243
1244         if ($media && $media eq 'cdrom') {
1245             $file = "local:iso/$file";
1246         } else {
1247             $file = "local:$vmid/$file";
1248         }
1249     }
1250
1251     return $file;
1252 }
1253
1254 sub verify_media_type {
1255     my ($opt, $vtype, $media) = @_;
1256
1257     return if !$media;
1258
1259     my $etype;
1260     if ($media eq 'disk') {
1261         $etype = 'images';
1262     } elsif ($media eq 'cdrom') {
1263         $etype = 'iso';
1264     } else {
1265         die "internal error";
1266     }
1267
1268     return if ($vtype eq $etype);
1269
1270     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1271 }
1272
1273 sub cleanup_drive_path {
1274     my ($opt, $storecfg, $drive) = @_;
1275
1276     # try to convert filesystem paths to volume IDs
1277
1278     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1279         ($drive->{file} !~ m|^/dev/.+|) &&
1280         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1281         ($drive->{file} !~ m/^\d+$/)) {
1282         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1283         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1284             if !$vtype;
1285         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1286         verify_media_type($opt, $vtype, $drive->{media});
1287         $drive->{file} = $volid;
1288     }
1289
1290     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1291 }
1292
1293 sub parse_hotplug_features {
1294     my ($data) = @_;
1295
1296     my $res = {};
1297
1298     return $res if $data eq '0';
1299
1300     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1301
1302     foreach my $feature (PVE::Tools::split_list($data)) {
1303         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304             $res->{$1} = 1;
1305         } else {
1306             die "invalid hotplug feature '$feature'\n";
1307         }
1308     }
1309     return $res;
1310 }
1311
1312 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1313 sub pve_verify_hotplug_features {
1314     my ($value, $noerr) = @_;
1315
1316     return $value if parse_hotplug_features($value);
1317
1318     return if $noerr;
1319
1320     die "unable to parse hotplug option\n";
1321 }
1322
1323 sub scsi_inquiry {
1324     my($fh, $noerr) = @_;
1325
1326     my $SG_IO = 0x2285;
1327     my $SG_GET_VERSION_NUM = 0x2282;
1328
1329     my $versionbuf = "\x00" x 8;
1330     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1331     if (!$ret) {
1332         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333         return;
1334     }
1335     my $version = unpack("I", $versionbuf);
1336     if ($version < 30000) {
1337         die "scsi generic interface too old\n"  if !$noerr;
1338         return;
1339     }
1340
1341     my $buf = "\x00" x 36;
1342     my $sensebuf = "\x00" x 8;
1343     my $cmd = pack("C x3 C x1", 0x12, 36);
1344
1345     # see /usr/include/scsi/sg.h
1346     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1347
1348     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1349                       length($sensebuf), 0, length($buf), $buf,
1350                       $cmd, $sensebuf, 6000);
1351
1352     $ret = ioctl($fh, $SG_IO, $packet);
1353     if (!$ret) {
1354         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1355         return;
1356     }
1357
1358     my @res = unpack($sg_io_hdr_t, $packet);
1359     if ($res[17] || $res[18]) {
1360         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1361         return;
1362     }
1363
1364     my $res = {};
1365     (my $byte0, my $byte1, $res->{vendor},
1366      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1367
1368     $res->{removable} = $byte1 & 128 ? 1 : 0;
1369     $res->{type} = $byte0 & 31;
1370
1371     return $res;
1372 }
1373
1374 sub path_is_scsi {
1375     my ($path) = @_;
1376
1377     my $fh = IO::File->new("+<$path") || return;
1378     my $res = scsi_inquiry($fh, 1);
1379     close($fh);
1380
1381     return $res;
1382 }
1383
1384 sub print_tabletdevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1388
1389     # we use uhci for old VMs because tablet driver was buggy in older qemu
1390     my $usbbus;
1391     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1392         $usbbus = 'ehci';
1393     } else {
1394         $usbbus = 'uhci';
1395     }
1396
1397     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 }
1399
1400 sub print_keyboarddevice_full {
1401     my ($conf, $arch, $machine) = @_;
1402
1403     return if $arch ne 'aarch64';
1404
1405     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 }
1407
1408 my sub get_drive_id {
1409     my ($drive) = @_;
1410     return "$drive->{interface}$drive->{index}";
1411 }
1412
1413 sub print_drivedevice_full {
1414     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1415
1416     my $device = '';
1417     my $maxdev = 0;
1418
1419     my $drive_id = get_drive_id($drive);
1420     if ($drive->{interface} eq 'virtio') {
1421         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1422         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1423         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1424     } elsif ($drive->{interface} eq 'scsi') {
1425
1426         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1427         my $unit = $drive->{index} % $maxdev;
1428         my $devicetype = 'hd';
1429         my $path = '';
1430         if (drive_is_cdrom($drive)) {
1431             $devicetype = 'cd';
1432         } else {
1433             if ($drive->{file} =~ m|^/|) {
1434                 $path = $drive->{file};
1435                 if (my $info = path_is_scsi($path)) {
1436                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1437                         $devicetype = 'block';
1438                     } elsif ($info->{type} == 1) { # tape
1439                         $devicetype = 'generic';
1440                     }
1441                 }
1442             } else {
1443                  $path = PVE::Storage::path($storecfg, $drive->{file});
1444             }
1445
1446             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1447             my $version = extract_version($machine_type, kvm_user_version());
1448             if ($path =~ m/^iscsi\:\/\// &&
1449                !min_version($version, 4, 1)) {
1450                 $devicetype = 'generic';
1451             }
1452         }
1453
1454         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1455             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1456         } else {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1458                 .",lun=$drive->{index}";
1459         }
1460         $device .= ",drive=drive-$drive_id,id=$drive_id";
1461
1462         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1463             $device .= ",rotation_rate=1";
1464         }
1465         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1466
1467     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1468         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1469         my $controller = int($drive->{index} / $maxdev);
1470         my $unit = $drive->{index} % $maxdev;
1471         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1472
1473         $device = "ide-$devicetype";
1474         if ($drive->{interface} eq 'ide') {
1475             $device .= ",bus=ide.$controller,unit=$unit";
1476         } else {
1477             $device .= ",bus=ahci$controller.$unit";
1478         }
1479         $device .= ",drive=drive-$drive_id,id=$drive_id";
1480
1481         if ($devicetype eq 'hd') {
1482             if (my $model = $drive->{model}) {
1483                 $model = URI::Escape::uri_unescape($model);
1484                 $device .= ",model=$model";
1485             }
1486             if ($drive->{ssd}) {
1487                 $device .= ",rotation_rate=1";
1488             }
1489         }
1490         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1491     } elsif ($drive->{interface} eq 'usb') {
1492         die "implement me";
1493         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1494     } else {
1495         die "unsupported interface type";
1496     }
1497
1498     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1499
1500     if (my $serial = $drive->{serial}) {
1501         $serial = URI::Escape::uri_unescape($serial);
1502         $device .= ",serial=$serial";
1503     }
1504
1505
1506     return $device;
1507 }
1508
1509 sub get_initiator_name {
1510     my $initiator;
1511
1512     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1513     while (defined(my $line = <$fh>)) {
1514         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1515         $initiator = $1;
1516         last;
1517     }
1518     $fh->close();
1519
1520     return $initiator;
1521 }
1522
1523 sub print_drive_commandline_full {
1524     my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1525
1526     my $path;
1527     my $volid = $drive->{file};
1528     my $format = $drive->{format};
1529     my $drive_id = get_drive_id($drive);
1530
1531     if (drive_is_cdrom($drive)) {
1532         $path = get_iso_path($storecfg, $vmid, $volid);
1533         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1534     } else {
1535         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1536         if ($storeid) {
1537             $path = PVE::Storage::path($storecfg, $volid);
1538             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1539             $format //= qemu_img_format($scfg, $volname);
1540         } else {
1541             $path = $volid;
1542             $format //= "raw";
1543         }
1544    }
1545
1546    my $is_rbd = $path =~ m/^rbd:/;
1547
1548     my $opts = '';
1549     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1550     foreach my $o (@qemu_drive_options) {
1551         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1552     }
1553
1554     # snapshot only accepts on|off
1555     if (defined($drive->{snapshot})) {
1556         my $v = $drive->{snapshot} ? 'on' : 'off';
1557         $opts .= ",snapshot=$v";
1558     }
1559
1560     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1561         my ($dir, $qmpname) = @$type;
1562         if (my $v = $drive->{"mbps$dir"}) {
1563             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1564         }
1565         if (my $v = $drive->{"mbps${dir}_max"}) {
1566             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1567         }
1568         if (my $v = $drive->{"bps${dir}_max_length"}) {
1569             $opts .= ",throttling.bps$qmpname-max-length=$v";
1570         }
1571         if (my $v = $drive->{"iops${dir}"}) {
1572             $opts .= ",throttling.iops$qmpname=$v";
1573         }
1574         if (my $v = $drive->{"iops${dir}_max"}) {
1575             $opts .= ",throttling.iops$qmpname-max=$v";
1576         }
1577         if (my $v = $drive->{"iops${dir}_max_length"}) {
1578             $opts .= ",throttling.iops$qmpname-max-length=$v";
1579         }
1580     }
1581
1582     if ($pbs_name) {
1583         $format = "rbd" if $is_rbd;
1584         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1585             if !$format;
1586         $opts .= ",format=alloc-track,file.driver=$format";
1587     } elsif ($format) {
1588         $opts .= ",format=$format";
1589     }
1590
1591     my $cache_direct = 0;
1592
1593     if (my $cache = $drive->{cache}) {
1594         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1595     } elsif (!drive_is_cdrom($drive)) {
1596         $opts .= ",cache=none";
1597         $cache_direct = 1;
1598     }
1599
1600     # aio native works only with O_DIRECT
1601     if (!$drive->{aio}) {
1602         if($cache_direct) {
1603             $opts .= ",aio=native";
1604         } else {
1605             $opts .= ",aio=threads";
1606         }
1607     }
1608
1609     if (!drive_is_cdrom($drive)) {
1610         my $detectzeroes;
1611         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1612             $detectzeroes = 'off';
1613         } elsif ($drive->{discard}) {
1614             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1615         } else {
1616             # This used to be our default with discard not being specified:
1617             $detectzeroes = 'on';
1618         }
1619
1620         # note: 'detect-zeroes' works per blockdev and we want it to persist
1621         # after the alloc-track is removed, so put it on 'file' directly
1622         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1623         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1624     }
1625
1626     if ($pbs_name) {
1627         $opts .= ",backing=$pbs_name";
1628         $opts .= ",auto-remove=on";
1629     }
1630
1631     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1632     my $file_param = "file";
1633     if ($pbs_name) {
1634         # non-rbd drivers require the underlying file to be a seperate block
1635         # node, so add a second .file indirection
1636         $file_param .= ".file" if !$is_rbd;
1637         $file_param .= ".filename";
1638     }
1639     my $pathinfo = $path ? "$file_param=$path," : '';
1640
1641     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1642 }
1643
1644 sub print_pbs_blockdev {
1645     my ($pbs_conf, $pbs_name) = @_;
1646     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1647     $blockdev .= ",repository=$pbs_conf->{repository}";
1648     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1649     $blockdev .= ",archive=$pbs_conf->{archive}";
1650     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1651     return $blockdev;
1652 }
1653
1654 sub print_netdevice_full {
1655     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1656
1657     my $device = $net->{model};
1658     if ($net->{model} eq 'virtio') {
1659          $device = 'virtio-net-pci';
1660      };
1661
1662     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1663     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1664     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1665         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1666         # and out of each queue plus one config interrupt and control vector queue
1667         my $vectors = $net->{queues} * 2 + 2;
1668         $tmpstr .= ",vectors=$vectors,mq=on";
1669     }
1670     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1671
1672     if (my $mtu = $net->{mtu}) {
1673         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1674             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1675             if ($mtu == 1) {
1676                  $mtu = $bridge_mtu;
1677             } elsif ($mtu < 576) {
1678                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1679             } elsif ($mtu > $bridge_mtu) {
1680                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1681             }
1682             $tmpstr .= ",host_mtu=$mtu";
1683         } else {
1684             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1685         }
1686     }
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     if (is_native($arch)) {
1725         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1726     }
1727
1728     my $vmname = $conf->{name} || "vm$vmid";
1729
1730     my $netdev = "";
1731     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733     if ($net->{bridge}) {
1734         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1735             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1736     } else {
1737         $netdev = "type=user,id=$netid,hostname=$vmname";
1738     }
1739
1740     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1741
1742     return $netdev;
1743 }
1744
1745 my $vga_map = {
1746     'cirrus' => 'cirrus-vga',
1747     'std' => 'VGA',
1748     'vmware' => 'vmware-svga',
1749     'virtio' => 'virtio-vga',
1750 };
1751
1752 sub print_vga_device {
1753     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1754
1755     my $type = $vga_map->{$vga->{type}};
1756     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1757         $type = 'virtio-gpu';
1758     }
1759     my $vgamem_mb = $vga->{memory};
1760
1761     my $max_outputs = '';
1762     if ($qxlnum) {
1763         $type = $id ? 'qxl' : 'qxl-vga';
1764
1765         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1766             # set max outputs so linux can have up to 4 qxl displays with one device
1767             if (min_version($machine_version, 4, 1)) {
1768                 $max_outputs = ",max_outputs=4";
1769             }
1770         }
1771     }
1772
1773     die "no devicetype for $vga->{type}\n" if !$type;
1774
1775     my $memory = "";
1776     if ($vgamem_mb) {
1777         if ($vga->{type} eq 'virtio') {
1778             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1779             $memory = ",max_hostmem=$bytes";
1780         } elsif ($qxlnum) {
1781             # from https://www.spice-space.org/multiple-monitors.html
1782             $memory = ",vgamem_mb=$vga->{memory}";
1783             my $ram = $vgamem_mb * 4;
1784             my $vram = $vgamem_mb * 2;
1785             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1786         } else {
1787             $memory = ",vgamem_mb=$vga->{memory}";
1788         }
1789     } elsif ($qxlnum && $id) {
1790         $memory = ",ram_size=67108864,vram_size=33554432";
1791     }
1792
1793     my $edidoff = "";
1794     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1795         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1796     }
1797
1798     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1799     my $vgaid = "vga" . ($id // '');
1800     my $pciaddr;
1801
1802     if ($q35 && $vgaid eq 'vga') {
1803         # the first display uses pcie.0 bus on q35 machines
1804         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1805     } else {
1806         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1807     }
1808
1809     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1810 }
1811
1812 sub parse_number_sets {
1813     my ($set) = @_;
1814     my $res = [];
1815     foreach my $part (split(/;/, $set)) {
1816         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818             push @$res, [ $1, $2 ];
1819         } else {
1820             die "invalid range: $part\n";
1821         }
1822     }
1823     return $res;
1824 }
1825
1826 sub parse_numa {
1827     my ($data) = @_;
1828
1829     my $res = parse_property_string($numa_fmt, $data);
1830     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1831     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1832     return $res;
1833 }
1834
1835 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1836 sub parse_net {
1837     my ($data) = @_;
1838
1839     my $res = eval { parse_property_string($net_fmt, $data) };
1840     if ($@) {
1841         warn $@;
1842         return;
1843     }
1844     if (!defined($res->{macaddr})) {
1845         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1846         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1847     }
1848     return $res;
1849 }
1850
1851 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1852 sub parse_ipconfig {
1853     my ($data) = @_;
1854
1855     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1856     if ($@) {
1857         warn $@;
1858         return;
1859     }
1860
1861     if ($res->{gw} && !$res->{ip}) {
1862         warn 'gateway specified without specifying an IP address';
1863         return;
1864     }
1865     if ($res->{gw6} && !$res->{ip6}) {
1866         warn 'IPv6 gateway specified without specifying an IPv6 address';
1867         return;
1868     }
1869     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1870         warn 'gateway specified together with DHCP';
1871         return;
1872     }
1873     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1874         # gw6 + auto/dhcp
1875         warn "IPv6 gateway specified together with $res->{ip6} address";
1876         return;
1877     }
1878
1879     if (!$res->{ip} && !$res->{ip6}) {
1880         return { ip => 'dhcp', ip6 => 'dhcp' };
1881     }
1882
1883     return $res;
1884 }
1885
1886 sub print_net {
1887     my $net = shift;
1888
1889     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1890 }
1891
1892 sub add_random_macs {
1893     my ($settings) = @_;
1894
1895     foreach my $opt (keys %$settings) {
1896         next if $opt !~ m/^net(\d+)$/;
1897         my $net = parse_net($settings->{$opt});
1898         next if !$net;
1899         $settings->{$opt} = print_net($net);
1900     }
1901 }
1902
1903 sub vm_is_volid_owner {
1904     my ($storecfg, $vmid, $volid) = @_;
1905
1906     if ($volid !~  m|^/|) {
1907         my ($path, $owner);
1908         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1909         if ($owner && ($owner == $vmid)) {
1910             return 1;
1911         }
1912     }
1913
1914     return;
1915 }
1916
1917 sub vmconfig_register_unused_drive {
1918     my ($storecfg, $vmid, $conf, $drive) = @_;
1919
1920     if (drive_is_cloudinit($drive)) {
1921         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1922         warn $@ if $@;
1923     } elsif (!drive_is_cdrom($drive)) {
1924         my $volid = $drive->{file};
1925         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1926             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1927         }
1928     }
1929 }
1930
1931 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1932 my $smbios1_fmt = {
1933     uuid => {
1934         type => 'string',
1935         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1936         format_description => 'UUID',
1937         description => "Set SMBIOS1 UUID.",
1938         optional => 1,
1939     },
1940     version => {
1941         type => 'string',
1942         pattern => '[A-Za-z0-9+\/]+={0,2}',
1943         format_description => 'Base64 encoded string',
1944         description => "Set SMBIOS1 version.",
1945         optional => 1,
1946     },
1947     serial => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 serial number.",
1952         optional => 1,
1953     },
1954     manufacturer => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 manufacturer.",
1959         optional => 1,
1960     },
1961     product => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 product ID.",
1966         optional => 1,
1967     },
1968     sku => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 SKU string.",
1973         optional => 1,
1974     },
1975     family => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 family string.",
1980         optional => 1,
1981     },
1982     base64 => {
1983         type => 'boolean',
1984         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1985         optional => 1,
1986     },
1987 };
1988
1989 sub parse_smbios1 {
1990     my ($data) = @_;
1991
1992     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1993     warn $@ if $@;
1994     return $res;
1995 }
1996
1997 sub print_smbios1 {
1998     my ($smbios1) = @_;
1999     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2000 }
2001
2002 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2003
2004 sub parse_watchdog {
2005     my ($value) = @_;
2006
2007     return if !$value;
2008
2009     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2010     warn $@ if $@;
2011     return $res;
2012 }
2013
2014 sub parse_guest_agent {
2015     my ($conf) = @_;
2016
2017     return {} if !defined($conf->{agent});
2018
2019     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2020     warn $@ if $@;
2021
2022     # if the agent is disabled ignore the other potentially set properties
2023     return {} if !$res->{enabled};
2024     return $res;
2025 }
2026
2027 sub get_qga_key {
2028     my ($conf, $key) = @_;
2029     return undef if !defined($conf->{agent});
2030
2031     my $agent = parse_guest_agent($conf);
2032     return $agent->{$key};
2033 }
2034
2035 sub parse_vga {
2036     my ($value) = @_;
2037
2038     return {} if !$value;
2039     my $res = eval { parse_property_string($vga_fmt, $value) };
2040     warn $@ if $@;
2041     return $res;
2042 }
2043
2044 sub parse_rng {
2045     my ($value) = @_;
2046
2047     return if !$value;
2048
2049     my $res = eval { parse_property_string($rng_fmt, $value) };
2050     warn $@ if $@;
2051     return $res;
2052 }
2053
2054 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2055 sub verify_usb_device {
2056     my ($value, $noerr) = @_;
2057
2058     return $value if parse_usb_device($value);
2059
2060     return if $noerr;
2061
2062     die "unable to parse usb device\n";
2063 }
2064
2065 # add JSON properties for create and set function
2066 sub json_config_properties {
2067     my $prop = shift;
2068
2069     foreach my $opt (keys %$confdesc) {
2070         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2071             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2072         $prop->{$opt} = $confdesc->{$opt};
2073     }
2074
2075     return $prop;
2076 }
2077
2078 # return copy of $confdesc_cloudinit to generate documentation
2079 sub cloudinit_config_properties {
2080
2081     return dclone($confdesc_cloudinit);
2082 }
2083
2084 sub check_type {
2085     my ($key, $value) = @_;
2086
2087     die "unknown setting '$key'\n" if !$confdesc->{$key};
2088
2089     my $type = $confdesc->{$key}->{type};
2090
2091     if (!defined($value)) {
2092         die "got undefined value\n";
2093     }
2094
2095     if ($value =~ m/[\n\r]/) {
2096         die "property contains a line feed\n";
2097     }
2098
2099     if ($type eq 'boolean') {
2100         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2101         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2102         die "type check ('boolean') failed - got '$value'\n";
2103     } elsif ($type eq 'integer') {
2104         return int($1) if $value =~ m/^(\d+)$/;
2105         die "type check ('integer') failed - got '$value'\n";
2106     } elsif ($type eq 'number') {
2107         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2108         die "type check ('number') failed - got '$value'\n";
2109     } elsif ($type eq 'string') {
2110         if (my $fmt = $confdesc->{$key}->{format}) {
2111             PVE::JSONSchema::check_format($fmt, $value);
2112             return $value;
2113         }
2114         $value =~ s/^\"(.*)\"$/$1/;
2115         return $value;
2116     } else {
2117         die "internal error"
2118     }
2119 }
2120
2121 sub destroy_vm {
2122     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2123
2124     my $conf = PVE::QemuConfig->load_config($vmid);
2125
2126     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2127
2128     if ($conf->{template}) {
2129         # check if any base image is still used by a linked clone
2130         PVE::QemuConfig->foreach_volume($conf, sub {
2131                 my ($ds, $drive) = @_;
2132                 return if drive_is_cdrom($drive);
2133
2134                 my $volid = $drive->{file};
2135                 return if !$volid || $volid =~ m|^/|;
2136
2137                 die "base volume '$volid' is still in use by linked cloned\n"
2138                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2139
2140         });
2141     }
2142
2143     # only remove disks owned by this VM (referenced in the config)
2144     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2145         my ($ds, $drive) = @_;
2146         return if drive_is_cdrom($drive, 1);
2147
2148         my $volid = $drive->{file};
2149         return if !$volid || $volid =~ m|^/|;
2150
2151         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2152         return if !$path || !$owner || ($owner != $vmid);
2153
2154         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2155         warn "Could not remove disk '$volid', check manually: $@" if $@;
2156     });
2157
2158     if ($purge_unreferenced) { # also remove unreferenced disk
2159         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2160         PVE::Storage::foreach_volid($vmdisks, sub {
2161             my ($volid, $sid, $volname, $d) = @_;
2162             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2163             warn $@ if $@;
2164         });
2165     }
2166
2167     if (defined $replacement_conf) {
2168         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2169     } else {
2170         PVE::QemuConfig->destroy_config($vmid);
2171     }
2172 }
2173
2174 sub parse_vm_config {
2175     my ($filename, $raw) = @_;
2176
2177     return if !defined($raw);
2178
2179     my $res = {
2180         digest => Digest::SHA::sha1_hex($raw),
2181         snapshots => {},
2182         pending => {},
2183     };
2184
2185     $filename =~ m|/qemu-server/(\d+)\.conf$|
2186         || die "got strange filename '$filename'";
2187
2188     my $vmid = $1;
2189
2190     my $conf = $res;
2191     my $descr;
2192     my $section = '';
2193
2194     my @lines = split(/\n/, $raw);
2195     foreach my $line (@lines) {
2196         next if $line =~ m/^\s*$/;
2197
2198         if ($line =~ m/^\[PENDING\]\s*$/i) {
2199             $section = 'pending';
2200             if (defined($descr)) {
2201                 $descr =~ s/\s+$//;
2202                 $conf->{description} = $descr;
2203             }
2204             $descr = undef;
2205             $conf = $res->{$section} = {};
2206             next;
2207
2208         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2209             $section = $1;
2210             if (defined($descr)) {
2211                 $descr =~ s/\s+$//;
2212                 $conf->{description} = $descr;
2213             }
2214             $descr = undef;
2215             $conf = $res->{snapshots}->{$section} = {};
2216             next;
2217         }
2218
2219         if ($line =~ m/^\#(.*)\s*$/) {
2220             $descr = '' if !defined($descr);
2221             $descr .= PVE::Tools::decode_text($1) . "\n";
2222             next;
2223         }
2224
2225         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2226             $descr = '' if !defined($descr);
2227             $descr .= PVE::Tools::decode_text($2);
2228         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2229             $conf->{snapstate} = $1;
2230         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2231             my $key = $1;
2232             my $value = $2;
2233             $conf->{$key} = $value;
2234         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2235             my $value = $1;
2236             if ($section eq 'pending') {
2237                 $conf->{delete} = $value; # we parse this later
2238             } else {
2239                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2240             }
2241         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2242             my $key = $1;
2243             my $value = $2;
2244             eval { $value = check_type($key, $value); };
2245             if ($@) {
2246                 warn "vm $vmid - unable to parse value of '$key' - $@";
2247             } else {
2248                 $key = 'ide2' if $key eq 'cdrom';
2249                 my $fmt = $confdesc->{$key}->{format};
2250                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2251                     my $v = parse_drive($key, $value);
2252                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2253                         $v->{file} = $volid;
2254                         $value = print_drive($v);
2255                     } else {
2256                         warn "vm $vmid - unable to parse value of '$key'\n";
2257                         next;
2258                     }
2259                 }
2260
2261                 $conf->{$key} = $value;
2262             }
2263         } else {
2264             warn "vm $vmid - unable to parse config: $line\n";
2265         }
2266     }
2267
2268     if (defined($descr)) {
2269         $descr =~ s/\s+$//;
2270         $conf->{description} = $descr;
2271     }
2272     delete $res->{snapstate}; # just to be sure
2273
2274     return $res;
2275 }
2276
2277 sub write_vm_config {
2278     my ($filename, $conf) = @_;
2279
2280     delete $conf->{snapstate}; # just to be sure
2281
2282     if ($conf->{cdrom}) {
2283         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2284         $conf->{ide2} = $conf->{cdrom};
2285         delete $conf->{cdrom};
2286     }
2287
2288     # we do not use 'smp' any longer
2289     if ($conf->{sockets}) {
2290         delete $conf->{smp};
2291     } elsif ($conf->{smp}) {
2292         $conf->{sockets} = $conf->{smp};
2293         delete $conf->{cores};
2294         delete $conf->{smp};
2295     }
2296
2297     my $used_volids = {};
2298
2299     my $cleanup_config = sub {
2300         my ($cref, $pending, $snapname) = @_;
2301
2302         foreach my $key (keys %$cref) {
2303             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2304                 $key eq 'snapstate' || $key eq 'pending';
2305             my $value = $cref->{$key};
2306             if ($key eq 'delete') {
2307                 die "propertry 'delete' is only allowed in [PENDING]\n"
2308                     if !$pending;
2309                 # fixme: check syntax?
2310                 next;
2311             }
2312             eval { $value = check_type($key, $value); };
2313             die "unable to parse value of '$key' - $@" if $@;
2314
2315             $cref->{$key} = $value;
2316
2317             if (!$snapname && is_valid_drivename($key)) {
2318                 my $drive = parse_drive($key, $value);
2319                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2320             }
2321         }
2322     };
2323
2324     &$cleanup_config($conf);
2325
2326     &$cleanup_config($conf->{pending}, 1);
2327
2328     foreach my $snapname (keys %{$conf->{snapshots}}) {
2329         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2330         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2331     }
2332
2333     # remove 'unusedX' settings if we re-add a volume
2334     foreach my $key (keys %$conf) {
2335         my $value = $conf->{$key};
2336         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2337             delete $conf->{$key};
2338         }
2339     }
2340
2341     my $generate_raw_config = sub {
2342         my ($conf, $pending) = @_;
2343
2344         my $raw = '';
2345
2346         # add description as comment to top of file
2347         if (defined(my $descr = $conf->{description})) {
2348             if ($descr) {
2349                 foreach my $cl (split(/\n/, $descr)) {
2350                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2351                 }
2352             } else {
2353                 $raw .= "#\n" if $pending;
2354             }
2355         }
2356
2357         foreach my $key (sort keys %$conf) {
2358             next if $key =~ /^(digest|description|pending|snapshots)$/;
2359             $raw .= "$key: $conf->{$key}\n";
2360         }
2361         return $raw;
2362     };
2363
2364     my $raw = &$generate_raw_config($conf);
2365
2366     if (scalar(keys %{$conf->{pending}})){
2367         $raw .= "\n[PENDING]\n";
2368         $raw .= &$generate_raw_config($conf->{pending}, 1);
2369     }
2370
2371     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2372         $raw .= "\n[$snapname]\n";
2373         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2374     }
2375
2376     return $raw;
2377 }
2378
2379 sub load_defaults {
2380
2381     my $res = {};
2382
2383     # we use static defaults from our JSON schema configuration
2384     foreach my $key (keys %$confdesc) {
2385         if (defined(my $default = $confdesc->{$key}->{default})) {
2386             $res->{$key} = $default;
2387         }
2388     }
2389
2390     return $res;
2391 }
2392
2393 sub config_list {
2394     my $vmlist = PVE::Cluster::get_vmlist();
2395     my $res = {};
2396     return $res if !$vmlist || !$vmlist->{ids};
2397     my $ids = $vmlist->{ids};
2398     my $nodename = nodename();
2399
2400     foreach my $vmid (keys %$ids) {
2401         my $d = $ids->{$vmid};
2402         next if !$d->{node} || $d->{node} ne $nodename;
2403         next if !$d->{type} || $d->{type} ne 'qemu';
2404         $res->{$vmid}->{exists} = 1;
2405     }
2406     return $res;
2407 }
2408
2409 # test if VM uses local resources (to prevent migration)
2410 sub check_local_resources {
2411     my ($conf, $noerr) = @_;
2412
2413     my @loc_res = ();
2414
2415     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2416     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2417
2418     push @loc_res, "ivshmem" if $conf->{ivshmem};
2419
2420     foreach my $k (keys %$conf) {
2421         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2422         # sockets are safe: they will recreated be on the target side post-migrate
2423         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2424         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2425     }
2426
2427     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2428
2429     return \@loc_res;
2430 }
2431
2432 # check if used storages are available on all nodes (use by migrate)
2433 sub check_storage_availability {
2434     my ($storecfg, $conf, $node) = @_;
2435
2436     PVE::QemuConfig->foreach_volume($conf, sub {
2437         my ($ds, $drive) = @_;
2438
2439         my $volid = $drive->{file};
2440         return if !$volid;
2441
2442         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2443         return if !$sid;
2444
2445         # check if storage is available on both nodes
2446         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2447         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2448    });
2449 }
2450
2451 # list nodes where all VM images are available (used by has_feature API)
2452 sub shared_nodes {
2453     my ($conf, $storecfg) = @_;
2454
2455     my $nodelist = PVE::Cluster::get_nodelist();
2456     my $nodehash = { map { $_ => 1 } @$nodelist };
2457     my $nodename = nodename();
2458
2459     PVE::QemuConfig->foreach_volume($conf, sub {
2460         my ($ds, $drive) = @_;
2461
2462         my $volid = $drive->{file};
2463         return if !$volid;
2464
2465         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2466         if ($storeid) {
2467             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2468             if ($scfg->{disable}) {
2469                 $nodehash = {};
2470             } elsif (my $avail = $scfg->{nodes}) {
2471                 foreach my $node (keys %$nodehash) {
2472                     delete $nodehash->{$node} if !$avail->{$node};
2473                 }
2474             } elsif (!$scfg->{shared}) {
2475                 foreach my $node (keys %$nodehash) {
2476                     delete $nodehash->{$node} if $node ne $nodename
2477                 }
2478             }
2479         }
2480     });
2481
2482     return $nodehash
2483 }
2484
2485 sub check_local_storage_availability {
2486     my ($conf, $storecfg) = @_;
2487
2488     my $nodelist = PVE::Cluster::get_nodelist();
2489     my $nodehash = { map { $_ => {} } @$nodelist };
2490
2491     PVE::QemuConfig->foreach_volume($conf, sub {
2492         my ($ds, $drive) = @_;
2493
2494         my $volid = $drive->{file};
2495         return if !$volid;
2496
2497         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2498         if ($storeid) {
2499             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2500
2501             if ($scfg->{disable}) {
2502                 foreach my $node (keys %$nodehash) {
2503                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2504                 }
2505             } elsif (my $avail = $scfg->{nodes}) {
2506                 foreach my $node (keys %$nodehash) {
2507                     if (!$avail->{$node}) {
2508                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2509                     }
2510                 }
2511             }
2512         }
2513     });
2514
2515     foreach my $node (values %$nodehash) {
2516         if (my $unavail = $node->{unavailable_storages}) {
2517             $node->{unavailable_storages} = [ sort keys %$unavail ];
2518         }
2519     }
2520
2521     return $nodehash
2522 }
2523
2524 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2525 sub check_running {
2526     my ($vmid, $nocheck, $node) = @_;
2527
2528     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2529     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2530 }
2531
2532 sub vzlist {
2533
2534     my $vzlist = config_list();
2535
2536     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2537
2538     while (defined(my $de = $fd->read)) {
2539         next if $de !~ m/^(\d+)\.pid$/;
2540         my $vmid = $1;
2541         next if !defined($vzlist->{$vmid});
2542         if (my $pid = check_running($vmid)) {
2543             $vzlist->{$vmid}->{pid} = $pid;
2544         }
2545     }
2546
2547     return $vzlist;
2548 }
2549
2550 our $vmstatus_return_properties = {
2551     vmid => get_standard_option('pve-vmid'),
2552     status => {
2553         description => "Qemu process status.",
2554         type => 'string',
2555         enum => ['stopped', 'running'],
2556     },
2557     maxmem => {
2558         description => "Maximum memory in bytes.",
2559         type => 'integer',
2560         optional => 1,
2561         renderer => 'bytes',
2562     },
2563     maxdisk => {
2564         description => "Root disk size in bytes.",
2565         type => 'integer',
2566         optional => 1,
2567         renderer => 'bytes',
2568     },
2569     name => {
2570         description => "VM name.",
2571         type => 'string',
2572         optional => 1,
2573     },
2574     qmpstatus => {
2575         description => "Qemu QMP agent status.",
2576         type => 'string',
2577         optional => 1,
2578     },
2579     pid => {
2580         description => "PID of running qemu process.",
2581         type => 'integer',
2582         optional => 1,
2583     },
2584     uptime => {
2585         description => "Uptime.",
2586         type => 'integer',
2587         optional => 1,
2588         renderer => 'duration',
2589     },
2590     cpus => {
2591         description => "Maximum usable CPUs.",
2592         type => 'number',
2593         optional => 1,
2594     },
2595     lock => {
2596         description => "The current config lock, if any.",
2597         type => 'string',
2598         optional => 1,
2599     },
2600     tags => {
2601         description  => "The current configured tags, if any",
2602         type => 'string',
2603         optional => 1,
2604     },
2605     'running-machine' => {
2606         description => "The currently running machine type (if running).",
2607         type => 'string',
2608         optional => 1,
2609     },
2610     'running-qemu' => {
2611         description => "The currently running QEMU version (if running).",
2612         type => 'string',
2613         optional => 1,
2614     },
2615 };
2616
2617 my $last_proc_pid_stat;
2618
2619 # get VM status information
2620 # This must be fast and should not block ($full == false)
2621 # We only query KVM using QMP if $full == true (this can be slow)
2622 sub vmstatus {
2623     my ($opt_vmid, $full) = @_;
2624
2625     my $res = {};
2626
2627     my $storecfg = PVE::Storage::config();
2628
2629     my $list = vzlist();
2630     my $defaults = load_defaults();
2631
2632     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2633
2634     my $cpucount = $cpuinfo->{cpus} || 1;
2635
2636     foreach my $vmid (keys %$list) {
2637         next if $opt_vmid && ($vmid ne $opt_vmid);
2638
2639         my $conf = PVE::QemuConfig->load_config($vmid);
2640
2641         my $d = { vmid => $vmid };
2642         $d->{pid} = $list->{$vmid}->{pid};
2643
2644         # fixme: better status?
2645         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2646
2647         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2648         if (defined($size)) {
2649             $d->{disk} = 0; # no info available
2650             $d->{maxdisk} = $size;
2651         } else {
2652             $d->{disk} = 0;
2653             $d->{maxdisk} = 0;
2654         }
2655
2656         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2657             * ($conf->{cores} || $defaults->{cores});
2658         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2659         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2660
2661         $d->{name} = $conf->{name} || "VM $vmid";
2662         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2663             : $defaults->{memory}*(1024*1024);
2664
2665         if ($conf->{balloon}) {
2666             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2667             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2668                 : $defaults->{shares};
2669         }
2670
2671         $d->{uptime} = 0;
2672         $d->{cpu} = 0;
2673         $d->{mem} = 0;
2674
2675         $d->{netout} = 0;
2676         $d->{netin} = 0;
2677
2678         $d->{diskread} = 0;
2679         $d->{diskwrite} = 0;
2680
2681         $d->{template} = PVE::QemuConfig->is_template($conf);
2682
2683         $d->{serial} = 1 if conf_has_serial($conf);
2684         $d->{lock} = $conf->{lock} if $conf->{lock};
2685         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2686
2687         $res->{$vmid} = $d;
2688     }
2689
2690     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2691     foreach my $dev (keys %$netdev) {
2692         next if $dev !~ m/^tap([1-9]\d*)i/;
2693         my $vmid = $1;
2694         my $d = $res->{$vmid};
2695         next if !$d;
2696
2697         $d->{netout} += $netdev->{$dev}->{receive};
2698         $d->{netin} += $netdev->{$dev}->{transmit};
2699
2700         if ($full) {
2701             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2702             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2703         }
2704
2705     }
2706
2707     my $ctime = gettimeofday;
2708
2709     foreach my $vmid (keys %$list) {
2710
2711         my $d = $res->{$vmid};
2712         my $pid = $d->{pid};
2713         next if !$pid;
2714
2715         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2716         next if !$pstat; # not running
2717
2718         my $used = $pstat->{utime} + $pstat->{stime};
2719
2720         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2721
2722         if ($pstat->{vsize}) {
2723             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2724         }
2725
2726         my $old = $last_proc_pid_stat->{$pid};
2727         if (!$old) {
2728             $last_proc_pid_stat->{$pid} = {
2729                 time => $ctime,
2730                 used => $used,
2731                 cpu => 0,
2732             };
2733             next;
2734         }
2735
2736         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2737
2738         if ($dtime > 1000) {
2739             my $dutime = $used -  $old->{used};
2740
2741             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2742             $last_proc_pid_stat->{$pid} = {
2743                 time => $ctime,
2744                 used => $used,
2745                 cpu => $d->{cpu},
2746             };
2747         } else {
2748             $d->{cpu} = $old->{cpu};
2749         }
2750     }
2751
2752     return $res if !$full;
2753
2754     my $qmpclient = PVE::QMPClient->new();
2755
2756     my $ballooncb = sub {
2757         my ($vmid, $resp) = @_;
2758
2759         my $info = $resp->{'return'};
2760         return if !$info->{max_mem};
2761
2762         my $d = $res->{$vmid};
2763
2764         # use memory assigned to VM
2765         $d->{maxmem} = $info->{max_mem};
2766         $d->{balloon} = $info->{actual};
2767
2768         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2769             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2770             $d->{freemem} = $info->{free_mem};
2771         }
2772
2773         $d->{ballooninfo} = $info;
2774     };
2775
2776     my $blockstatscb = sub {
2777         my ($vmid, $resp) = @_;
2778         my $data = $resp->{'return'} || [];
2779         my $totalrdbytes = 0;
2780         my $totalwrbytes = 0;
2781
2782         for my $blockstat (@$data) {
2783             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2784             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2785
2786             $blockstat->{device} =~ s/drive-//;
2787             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2788         }
2789         $res->{$vmid}->{diskread} = $totalrdbytes;
2790         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2791     };
2792
2793     my $machinecb = sub {
2794         my ($vmid, $resp) = @_;
2795         my $data = $resp->{'return'} || [];
2796
2797         $res->{$vmid}->{'running-machine'} =
2798             PVE::QemuServer::Machine::current_from_query_machines($data);
2799     };
2800
2801     my $versioncb = sub {
2802         my ($vmid, $resp) = @_;
2803         my $data = $resp->{'return'} // {};
2804         my $version = 'unknown';
2805
2806         if (my $v = $data->{qemu}) {
2807             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2808         }
2809
2810         $res->{$vmid}->{'running-qemu'} = $version;
2811     };
2812
2813     my $statuscb = sub {
2814         my ($vmid, $resp) = @_;
2815
2816         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2817         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2818         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2819         # this fails if ballon driver is not loaded, so this must be
2820         # the last commnand (following command are aborted if this fails).
2821         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2822
2823         my $status = 'unknown';
2824         if (!defined($status = $resp->{'return'}->{status})) {
2825             warn "unable to get VM status\n";
2826             return;
2827         }
2828
2829         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2830     };
2831
2832     foreach my $vmid (keys %$list) {
2833         next if $opt_vmid && ($vmid ne $opt_vmid);
2834         next if !$res->{$vmid}->{pid}; # not running
2835         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2836     }
2837
2838     $qmpclient->queue_execute(undef, 2);
2839
2840     foreach my $vmid (keys %$list) {
2841         next if $opt_vmid && ($vmid ne $opt_vmid);
2842         next if !$res->{$vmid}->{pid}; #not running
2843
2844         # we can't use the $qmpclient since it might have already aborted on
2845         # 'query-balloon', but this might also fail for older versions...
2846         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2847         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2848     }
2849
2850     foreach my $vmid (keys %$list) {
2851         next if $opt_vmid && ($vmid ne $opt_vmid);
2852         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2853     }
2854
2855     return $res;
2856 }
2857
2858 sub conf_has_serial {
2859     my ($conf) = @_;
2860
2861     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2862         if ($conf->{"serial$i"}) {
2863             return 1;
2864         }
2865     }
2866
2867     return 0;
2868 }
2869
2870 sub conf_has_audio {
2871     my ($conf, $id) = @_;
2872
2873     $id //= 0;
2874     my $audio = $conf->{"audio$id"};
2875     return if !defined($audio);
2876
2877     my $audioproperties = parse_property_string($audio_fmt, $audio);
2878     my $audiodriver = $audioproperties->{driver} // 'spice';
2879
2880     return {
2881         dev => $audioproperties->{device},
2882         dev_id => "audiodev$id",
2883         backend => $audiodriver,
2884         backend_id => "$audiodriver-backend${id}",
2885     };
2886 }
2887
2888 sub audio_devs {
2889     my ($audio, $audiopciaddr, $machine_version) = @_;
2890
2891     my $devs = [];
2892
2893     my $id = $audio->{dev_id};
2894     my $audiodev = "";
2895     if (min_version($machine_version, 4, 2)) {
2896         $audiodev = ",audiodev=$audio->{backend_id}";
2897     }
2898
2899     if ($audio->{dev} eq 'AC97') {
2900         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2901     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2902         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2903         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2904         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2905     } else {
2906         die "unkown audio device '$audio->{dev}', implement me!";
2907     }
2908
2909     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2910
2911     return $devs;
2912 }
2913
2914 sub vga_conf_has_spice {
2915     my ($vga) = @_;
2916
2917     my $vgaconf = parse_vga($vga);
2918     my $vgatype = $vgaconf->{type};
2919     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2920
2921     return $1 || 1;
2922 }
2923
2924 sub is_native($) {
2925     my ($arch) = @_;
2926     return get_host_arch() eq $arch;
2927 }
2928
2929 sub get_vm_arch {
2930     my ($conf) = @_;
2931     return $conf->{arch} // get_host_arch();
2932 }
2933
2934 my $default_machines = {
2935     x86_64 => 'pc',
2936     aarch64 => 'virt',
2937 };
2938
2939 sub get_installed_machine_version {
2940     my ($kvmversion) = @_;
2941     $kvmversion = kvm_user_version() if !defined($kvmversion);
2942     $kvmversion =~ m/^(\d+\.\d+)/;
2943     return $1;
2944 }
2945
2946 sub windows_get_pinned_machine_version {
2947     my ($machine, $base_version, $kvmversion) = @_;
2948
2949     my $pin_version = $base_version;
2950     if (!defined($base_version) ||
2951         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2952     ) {
2953         $pin_version = get_installed_machine_version($kvmversion);
2954     }
2955     if (!$machine || $machine eq 'pc') {
2956         $machine = "pc-i440fx-$pin_version";
2957     } elsif ($machine eq 'q35') {
2958         $machine = "pc-q35-$pin_version";
2959     } elsif ($machine eq 'virt') {
2960         $machine = "virt-$pin_version";
2961     } else {
2962         warn "unknown machine type '$machine', not touching that!\n";
2963     }
2964
2965     return $machine;
2966 }
2967
2968 sub get_vm_machine {
2969     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2970
2971     my $machine = $forcemachine || $conf->{machine};
2972
2973     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2974         $kvmversion //= kvm_user_version();
2975         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2976         # layout which confuses windows quite a bit and may result in various regressions..
2977         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2978         if (windows_version($conf->{ostype})) {
2979             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2980         }
2981         $arch //= 'x86_64';
2982         $machine ||= $default_machines->{$arch};
2983         if ($add_pve_version) {
2984             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2985             $machine .= "+pve$pvever";
2986         }
2987     }
2988
2989     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
2990         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
2991         $machine = $1 if $is_pxe;
2992
2993         # for version-pinned machines that do not include a pve-version (e.g.
2994         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2995         $machine .= '+pve0';
2996
2997         $machine .= '.pxe' if $is_pxe;
2998     }
2999
3000     return $machine;
3001 }
3002
3003 sub get_ovmf_files($) {
3004     my ($arch) = @_;
3005
3006     my $ovmf = $OVMF->{$arch}
3007         or die "no OVMF images known for architecture '$arch'\n";
3008
3009     return @$ovmf;
3010 }
3011
3012 my $Arch2Qemu = {
3013     aarch64 => '/usr/bin/qemu-system-aarch64',
3014     x86_64 => '/usr/bin/qemu-system-x86_64',
3015 };
3016 sub get_command_for_arch($) {
3017     my ($arch) = @_;
3018     return '/usr/bin/kvm' if is_native($arch);
3019
3020     my $cmd = $Arch2Qemu->{$arch}
3021         or die "don't know how to emulate architecture '$arch'\n";
3022     return $cmd;
3023 }
3024
3025 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3026 # to use in a QEMU command line (-cpu element), first array_intersect the result
3027 # of query_supported_ with query_understood_. This is necessary because:
3028 #
3029 # a) query_understood_ returns flags the host cannot use and
3030 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3031 #    flags, but CPU settings - with most of them being flags. Those settings
3032 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3033 #
3034 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3035 # expensive. If you need the value returned from this, you can get it much
3036 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3037 # $accel being 'kvm' or 'tcg'.
3038 #
3039 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3040 # changes, automatically populating pmxcfs.
3041 #
3042 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3043 # since kvm and tcg machines support different flags
3044 #
3045 sub query_supported_cpu_flags {
3046     my ($arch) = @_;
3047
3048     $arch //= get_host_arch();
3049     my $default_machine = $default_machines->{$arch};
3050
3051     my $flags = {};
3052
3053     # FIXME: Once this is merged, the code below should work for ARM as well:
3054     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3055     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3056         $arch eq "aarch64";
3057
3058     my $kvm_supported = defined(kvm_version());
3059     my $qemu_cmd = get_command_for_arch($arch);
3060     my $fakevmid = -1;
3061     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3062
3063     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3064     my $query_supported_run_qemu = sub {
3065         my ($kvm) = @_;
3066
3067         my $flags = {};
3068         my $cmd = [
3069             $qemu_cmd,
3070             '-machine', $default_machine,
3071             '-display', 'none',
3072             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3073             '-mon', 'chardev=qmp,mode=control',
3074             '-pidfile', $pidfile,
3075             '-S', '-daemonize'
3076         ];
3077
3078         if (!$kvm) {
3079             push @$cmd, '-accel', 'tcg';
3080         }
3081
3082         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3083         die "QEMU flag querying VM exited with code " . $rc if $rc;
3084
3085         eval {
3086             my $cmd_result = mon_cmd(
3087                 $fakevmid,
3088                 'query-cpu-model-expansion',
3089                 type => 'full',
3090                 model => { name => 'host' }
3091             );
3092
3093             my $props = $cmd_result->{model}->{props};
3094             foreach my $prop (keys %$props) {
3095                 next if $props->{$prop} ne '1';
3096                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3097                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3098                 # We only keep those with underscores, to match /proc/cpuinfo
3099                 $prop =~ s/\.|-/_/g;
3100                 $flags->{$prop} = 1;
3101             }
3102         };
3103         my $err = $@;
3104
3105         # force stop with 10 sec timeout and 'nocheck'
3106         # always stop, even if QMP failed
3107         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3108
3109         die $err if $err;
3110
3111         return [ sort keys %$flags ];
3112     };
3113
3114     # We need to query QEMU twice, since KVM and TCG have different supported flags
3115     PVE::QemuConfig->lock_config($fakevmid, sub {
3116         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3117         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3118
3119         if ($kvm_supported) {
3120             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3121             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3122         }
3123     });
3124
3125     return $flags;
3126 }
3127
3128 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3129 my $understood_cpu_flag_dir = "/usr/share/kvm";
3130 sub query_understood_cpu_flags {
3131     my $arch = get_host_arch();
3132     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3133
3134     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3135         if ! -e $filepath;
3136
3137     my $raw = file_get_contents($filepath);
3138     $raw =~ s/^\s+|\s+$//g;
3139     my @flags = split(/\s+/, $raw);
3140
3141     return \@flags;
3142 }
3143
3144 sub config_to_command {
3145     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3146         $pbs_backing) = @_;
3147
3148     my $cmd = [];
3149     my $globalFlags = [];
3150     my $machineFlags = [];
3151     my $rtcFlags = [];
3152     my $devices = [];
3153     my $pciaddr = '';
3154     my $bridges = {};
3155     my $ostype = $conf->{ostype};
3156     my $winversion = windows_version($ostype);
3157     my $kvm = $conf->{kvm};
3158     my $nodename = nodename();
3159
3160     my $arch = get_vm_arch($conf);
3161     my $kvm_binary = get_command_for_arch($arch);
3162     my $kvmver = kvm_user_version($kvm_binary);
3163
3164     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3165         $kvmver //= "undefined";
3166         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3167     }
3168
3169     my $add_pve_version = min_version($kvmver, 4, 1);
3170
3171     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3172     my $machine_version = extract_version($machine_type, $kvmver);
3173     $kvm //= 1 if is_native($arch);
3174
3175     $machine_version =~ m/(\d+)\.(\d+)/;
3176     my ($machine_major, $machine_minor) = ($1, $2);
3177
3178     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3179         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3180     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3181         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3182             ." please upgrade node '$nodename'\n"
3183     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3184         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3185         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3186             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3187             ." node '$nodename'\n";
3188     }
3189
3190     # if a specific +pve version is required for a feature, use $version_guard
3191     # instead of min_version to allow machines to be run with the minimum
3192     # required version
3193     my $required_pve_version = 0;
3194     my $version_guard = sub {
3195         my ($major, $minor, $pve) = @_;
3196         return 0 if !min_version($machine_version, $major, $minor, $pve);
3197         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3198         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3199         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3200         return 1;
3201     };
3202
3203     if ($kvm && !defined kvm_version()) {
3204         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3205             ." or enable in BIOS.\n";
3206     }
3207
3208     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3209     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3210     my $use_old_bios_files = undef;
3211     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3212
3213     my $cpuunits = defined($conf->{cpuunits}) ?
3214             $conf->{cpuunits} : $defaults->{cpuunits};
3215
3216     push @$cmd, $kvm_binary;
3217
3218     push @$cmd, '-id', $vmid;
3219
3220     my $vmname = $conf->{name} || "vm$vmid";
3221
3222     push @$cmd, '-name', $vmname;
3223
3224     push @$cmd, '-no-shutdown';
3225
3226     my $use_virtio = 0;
3227
3228     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3229     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3230     push @$cmd, '-mon', "chardev=qmp,mode=control";
3231
3232     if (min_version($machine_version, 2, 12)) {
3233         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3234         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3235     }
3236
3237     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3238
3239     push @$cmd, '-daemonize';
3240
3241     if ($conf->{smbios1}) {
3242         my $smbios_conf = parse_smbios1($conf->{smbios1});
3243         if ($smbios_conf->{base64}) {
3244             # Do not pass base64 flag to qemu
3245             delete $smbios_conf->{base64};
3246             my $smbios_string = "";
3247             foreach my $key (keys %$smbios_conf) {
3248                 my $value;
3249                 if ($key eq "uuid") {
3250                     $value = $smbios_conf->{uuid}
3251                 } else {
3252                     $value = decode_base64($smbios_conf->{$key});
3253                 }
3254                 # qemu accepts any binary data, only commas need escaping by double comma
3255                 $value =~ s/,/,,/g;
3256                 $smbios_string .= "," . $key . "=" . $value if $value;
3257             }
3258             push @$cmd, '-smbios', "type=1" . $smbios_string;
3259         } else {
3260             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3261         }
3262     }
3263
3264     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3265         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3266         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3267
3268         my ($path, $format);
3269         if (my $efidisk = $conf->{efidisk0}) {
3270             my $d = parse_drive('efidisk0', $efidisk);
3271             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3272             $format = $d->{format};
3273             if ($storeid) {
3274                 $path = PVE::Storage::path($storecfg, $d->{file});
3275                 if (!defined($format)) {
3276                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3277                     $format = qemu_img_format($scfg, $volname);
3278                 }
3279             } else {
3280                 $path = $d->{file};
3281                 die "efidisk format must be specified\n"
3282                     if !defined($format);
3283             }
3284         } else {
3285             warn "no efidisk configured! Using temporary efivars disk.\n";
3286             $path = "/tmp/$vmid-ovmf.fd";
3287             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3288             $format = 'raw';
3289         }
3290
3291         my $size_str = "";
3292
3293         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3294             $size_str = ",size=" . (-s $ovmf_vars);
3295         }
3296
3297         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3298         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3299     }
3300
3301     # load q35 config
3302     if ($q35) {
3303         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3304         if (min_version($machine_version, 4, 0)) {
3305             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3306         } else {
3307             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3308         }
3309     }
3310
3311     if ($conf->{vmgenid}) {
3312         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3313     }
3314
3315     # add usb controllers
3316     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3317         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3318     push @$devices, @usbcontrollers if @usbcontrollers;
3319     my $vga = parse_vga($conf->{vga});
3320
3321     my $qxlnum = vga_conf_has_spice($conf->{vga});
3322     $vga->{type} = 'qxl' if $qxlnum;
3323
3324     if (!$vga->{type}) {
3325         if ($arch eq 'aarch64') {
3326             $vga->{type} = 'virtio';
3327         } elsif (min_version($machine_version, 2, 9)) {
3328             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3329         } else {
3330             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3331         }
3332     }
3333
3334     # enable absolute mouse coordinates (needed by vnc)
3335     my $tablet;
3336     if (defined($conf->{tablet})) {
3337         $tablet = $conf->{tablet};
3338     } else {
3339         $tablet = $defaults->{tablet};
3340         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3341         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3342     }
3343
3344     if ($tablet) {
3345         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3346         my $kbd = print_keyboarddevice_full($conf, $arch);
3347         push @$devices, '-device', $kbd if defined($kbd);
3348     }
3349
3350     my $bootorder = device_bootorder($conf);
3351
3352     # host pci device passthrough
3353     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3354         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3355
3356     # usb devices
3357     my $usb_dev_features = {};
3358     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3359
3360     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3361         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3362     push @$devices, @usbdevices if @usbdevices;
3363
3364     # serial devices
3365     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3366         if (my $path = $conf->{"serial$i"}) {
3367             if ($path eq 'socket') {
3368                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3369                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3370                 # On aarch64, serial0 is the UART device. Qemu only allows
3371                 # connecting UART devices via the '-serial' command line, as
3372                 # the device has a fixed slot on the hardware...
3373                 if ($arch eq 'aarch64' && $i == 0) {
3374                     push @$devices, '-serial', "chardev:serial$i";
3375                 } else {
3376                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3377                 }
3378             } else {
3379                 die "no such serial device\n" if ! -c $path;
3380                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3381                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3382             }
3383         }
3384     }
3385
3386     # parallel devices
3387     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3388         if (my $path = $conf->{"parallel$i"}) {
3389             die "no such parallel device\n" if ! -c $path;
3390             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3391             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3392             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3393         }
3394     }
3395
3396     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3397         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3398         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3399         push @$devices, @$audio_devs;
3400     }
3401
3402     my $sockets = 1;
3403     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3404     $sockets = $conf->{sockets} if  $conf->{sockets};
3405
3406     my $cores = $conf->{cores} || 1;
3407
3408     my $maxcpus = $sockets * $cores;
3409
3410     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3411
3412     my $allowed_vcpus = $cpuinfo->{cpus};
3413
3414     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3415         if ($allowed_vcpus < $maxcpus);
3416
3417     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3418
3419         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3420         for (my $i = 2; $i <= $vcpus; $i++)  {
3421             my $cpustr = print_cpu_device($conf,$i);
3422             push @$cmd, '-device', $cpustr;
3423         }
3424
3425     } else {
3426
3427         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3428     }
3429     push @$cmd, '-nodefaults';
3430
3431     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3432
3433     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3434
3435     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3436
3437     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3438         push @$devices, '-device', print_vga_device(
3439             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3440         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3441         push @$cmd,  '-vnc', "unix:$socket,password";
3442     } else {
3443         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3444         push @$cmd, '-nographic';
3445     }
3446
3447     # time drift fix
3448     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3449     my $useLocaltime = $conf->{localtime};
3450
3451     if ($winversion >= 5) { # windows
3452         $useLocaltime = 1 if !defined($conf->{localtime});
3453
3454         # use time drift fix when acpi is enabled
3455         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3456             $tdf = 1 if !defined($conf->{tdf});
3457         }
3458     }
3459
3460     if ($winversion >= 6) {
3461         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3462         push @$cmd, '-no-hpet';
3463     }
3464
3465     push @$rtcFlags, 'driftfix=slew' if $tdf;
3466
3467     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3468         push @$rtcFlags, "base=$conf->{startdate}";
3469     } elsif ($useLocaltime) {
3470         push @$rtcFlags, 'base=localtime';
3471     }
3472
3473     if ($forcecpu) {
3474         push @$cmd, '-cpu', $forcecpu;
3475     } else {
3476         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3477     }
3478
3479     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3480
3481     push @$cmd, '-S' if $conf->{freeze};
3482
3483     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3484
3485     my $guest_agent = parse_guest_agent($conf);
3486
3487     if ($guest_agent->{enabled}) {
3488         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3489         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3490
3491         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3492             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3493             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3494             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3495         } elsif ($guest_agent->{type} eq 'isa') {
3496             push @$devices, '-device', "isa-serial,chardev=qga0";
3497         }
3498     }
3499
3500     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3501     if ($rng && $version_guard->(4, 1, 2)) {
3502         check_rng_source($rng->{source});
3503
3504         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3505         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3506         my $limiter_str = "";
3507         if ($max_bytes) {
3508             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3509         }
3510
3511         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3512         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3513         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3514     }
3515
3516     my $spice_port;
3517
3518     if ($qxlnum) {
3519         if ($qxlnum > 1) {
3520             if ($winversion){
3521                 for (my $i = 1; $i < $qxlnum; $i++){
3522                     push @$devices, '-device', print_vga_device(
3523                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3524                 }
3525             } else {
3526                 # assume other OS works like Linux
3527                 my ($ram, $vram) = ("134217728", "67108864");
3528                 if ($vga->{memory}) {
3529                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3530                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3531                 }
3532                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3533                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3534             }
3535         }
3536
3537         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3538
3539         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3540         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3541         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3542
3543         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3544         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3545         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3546
3547         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3548         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3549
3550         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3551         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3552         if ($spice_enhancement->{foldersharing}) {
3553             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3554             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3555         }
3556
3557         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3558         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3559             if $spice_enhancement->{videostreaming};
3560
3561         push @$devices, '-spice', "$spice_opts";
3562     }
3563
3564     # enable balloon by default, unless explicitly disabled
3565     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3566         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3567         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3568     }
3569
3570     if ($conf->{watchdog}) {
3571         my $wdopts = parse_watchdog($conf->{watchdog});
3572         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3573         my $watchdog = $wdopts->{model} || 'i6300esb';
3574         push @$devices, '-device', "$watchdog$pciaddr";
3575         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3576     }
3577
3578     my $vollist = [];
3579     my $scsicontroller = {};
3580     my $ahcicontroller = {};
3581     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3582
3583     # Add iscsi initiator name if available
3584     if (my $initiator = get_initiator_name()) {
3585         push @$devices, '-iscsi', "initiator-name=$initiator";
3586     }
3587
3588     PVE::QemuConfig->foreach_volume($conf, sub {
3589         my ($ds, $drive) = @_;
3590
3591         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3592             push @$vollist, $drive->{file};
3593         }
3594
3595         # ignore efidisk here, already added in bios/fw handling code above
3596         return if $drive->{interface} eq 'efidisk';
3597
3598         $use_virtio = 1 if $ds =~ m/^virtio/;
3599
3600         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3601
3602         if ($drive->{interface} eq 'virtio'){
3603            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3604         }
3605
3606         if ($drive->{interface} eq 'scsi') {
3607
3608             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3609
3610             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3611                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3612
3613             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3614             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3615
3616             my $iothread = '';
3617             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3618                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3619                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3620             } elsif ($drive->{iothread}) {
3621                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3622             }
3623
3624             my $queues = '';
3625             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3626                 $queues = ",num_queues=$drive->{queues}";
3627             }
3628
3629             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3630                 if !$scsicontroller->{$controller};
3631             $scsicontroller->{$controller}=1;
3632         }
3633
3634         if ($drive->{interface} eq 'sata') {
3635             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3636             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3637             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3638                 if !$ahcicontroller->{$controller};
3639             $ahcicontroller->{$controller}=1;
3640         }
3641
3642         my $pbs_conf = $pbs_backing->{$ds};
3643         my $pbs_name = undef;
3644         if ($pbs_conf) {
3645             $pbs_name = "drive-$ds-pbs";
3646             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3647         }
3648
3649         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3650
3651         # extra protection for templates, but SATA and IDE don't support it..
3652         my $read_only = PVE::QemuConfig->is_template($conf)
3653             && $drive->{interface} ne 'sata'
3654             && $drive->{interface} ne 'ide';
3655
3656         $drive_cmd .= ',readonly' if $read_only;
3657
3658         push @$devices, '-drive',$drive_cmd;
3659         push @$devices, '-device', print_drivedevice_full(
3660             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3661     });
3662
3663     for (my $i = 0; $i < $MAX_NETS; $i++) {
3664         my $netname = "net$i";
3665
3666         next if !$conf->{$netname};
3667         my $d = parse_net($conf->{$netname});
3668         next if !$d;
3669
3670         $use_virtio = 1 if $d->{model} eq 'virtio';
3671
3672         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3673
3674         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3675         push @$devices, '-netdev', $netdevfull;
3676
3677         my $netdevicefull = print_netdevice_full(
3678             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3679
3680         push @$devices, '-device', $netdevicefull;
3681     }
3682
3683     if ($conf->{ivshmem}) {
3684         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3685
3686         my $bus;
3687         if ($q35) {
3688             $bus = print_pcie_addr("ivshmem");
3689         } else {
3690             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3691         }
3692
3693         my $ivshmem_name = $ivshmem->{name} // $vmid;
3694         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3695
3696         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3697         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3698             .",size=$ivshmem->{size}M";
3699     }
3700
3701     # pci.4 is nested in pci.1
3702     $bridges->{1} = 1 if $bridges->{4};
3703
3704     if (!$q35) {
3705         # add pci bridges
3706         if (min_version($machine_version, 2, 3)) {
3707            $bridges->{1} = 1;
3708            $bridges->{2} = 1;
3709         }
3710
3711         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3712
3713     }
3714
3715     for my $k (sort {$b cmp $a} keys %$bridges) {
3716         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3717
3718         my $k_name = $k;
3719         if ($k == 2 && $legacy_igd) {
3720             $k_name = "$k-igd";
3721         }
3722         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3723
3724         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3725         if ($q35) {
3726             # add after -readconfig pve-q35.cfg
3727             splice @$devices, 2, 0, '-device', $devstr;
3728         } else {
3729             unshift @$devices, '-device', $devstr if $k > 0;
3730         }
3731     }
3732
3733     if (!$kvm) {
3734         push @$machineFlags, 'accel=tcg';
3735     }
3736
3737     my $machine_type_min = $machine_type;
3738     if ($add_pve_version) {
3739         $machine_type_min =~ s/\+pve\d+$//;
3740         $machine_type_min .= "+pve$required_pve_version";
3741     }
3742     push @$machineFlags, "type=${machine_type_min}";
3743
3744     push @$cmd, @$devices;
3745     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3746     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3747     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3748
3749     if (my $vmstate = $conf->{vmstate}) {
3750         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3751         push @$vollist, $vmstate;
3752         push @$cmd, '-loadstate', $statepath;
3753         print "activating and using '$vmstate' as vmstate\n";
3754     }
3755
3756     # add custom args
3757     if ($conf->{args}) {
3758         my $aa = PVE::Tools::split_args($conf->{args});
3759         push @$cmd, @$aa;
3760     }
3761
3762     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3763 }
3764
3765 sub check_rng_source {
3766     my ($source) = @_;
3767
3768     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3769     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3770         if ! -e $source;
3771
3772     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3773     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3774         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3775         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3776         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3777             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3778             ." to the host.\n";
3779     }
3780 }
3781
3782 sub spice_port {
3783     my ($vmid) = @_;
3784
3785     my $res = mon_cmd($vmid, 'query-spice');
3786
3787     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3788 }
3789
3790 sub vm_devices_list {
3791     my ($vmid) = @_;
3792
3793     my $res = mon_cmd($vmid, 'query-pci');
3794     my $devices_to_check = [];
3795     my $devices = {};
3796     foreach my $pcibus (@$res) {
3797         push @$devices_to_check, @{$pcibus->{devices}},
3798     }
3799
3800     while (@$devices_to_check) {
3801         my $to_check = [];
3802         for my $d (@$devices_to_check) {
3803             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3804             next if !$d->{'pci_bridge'};
3805
3806             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3807             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3808         }
3809         $devices_to_check = $to_check;
3810     }
3811
3812     my $resblock = mon_cmd($vmid, 'query-block');
3813     foreach my $block (@$resblock) {
3814         if($block->{device} =~ m/^drive-(\S+)/){
3815                 $devices->{$1} = 1;
3816         }
3817     }
3818
3819     my $resmice = mon_cmd($vmid, 'query-mice');
3820     foreach my $mice (@$resmice) {
3821         if ($mice->{name} eq 'QEMU HID Tablet') {
3822             $devices->{tablet} = 1;
3823             last;
3824         }
3825     }
3826
3827     # for usb devices there is no query-usb
3828     # but we can iterate over the entries in
3829     # qom-list path=/machine/peripheral
3830     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3831     foreach my $per (@$resperipheral) {
3832         if ($per->{name} =~ m/^usb\d+$/) {
3833             $devices->{$per->{name}} = 1;
3834         }
3835     }
3836
3837     return $devices;
3838 }
3839
3840 sub vm_deviceplug {
3841     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3842
3843     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3844
3845     my $devices_list = vm_devices_list($vmid);
3846     return 1 if defined($devices_list->{$deviceid});
3847
3848     # add PCI bridge if we need it for the device
3849     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3850
3851     if ($deviceid eq 'tablet') {
3852
3853         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3854
3855     } elsif ($deviceid eq 'keyboard') {
3856
3857         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3858
3859     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3860
3861         die "usb hotplug currently not reliable\n";
3862         # since we can't reliably hot unplug all added usb devices and usb
3863         # passthrough breaks live migration we disable usb hotplugging for now
3864         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3865
3866     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3867
3868         qemu_iothread_add($vmid, $deviceid, $device);
3869
3870         qemu_driveadd($storecfg, $vmid, $device);
3871         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3872
3873         qemu_deviceadd($vmid, $devicefull);
3874         eval { qemu_deviceaddverify($vmid, $deviceid); };
3875         if (my $err = $@) {
3876             eval { qemu_drivedel($vmid, $deviceid); };
3877             warn $@ if $@;
3878             die $err;
3879         }
3880
3881     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3882
3883
3884         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3885         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3886         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3887
3888         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3889
3890         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3891             qemu_iothread_add($vmid, $deviceid, $device);
3892             $devicefull .= ",iothread=iothread-$deviceid";
3893         }
3894
3895         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3896             $devicefull .= ",num_queues=$device->{queues}";
3897         }
3898
3899         qemu_deviceadd($vmid, $devicefull);
3900         qemu_deviceaddverify($vmid, $deviceid);
3901
3902     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3903
3904         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3905         qemu_driveadd($storecfg, $vmid, $device);
3906
3907         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3908         eval { qemu_deviceadd($vmid, $devicefull); };
3909         if (my $err = $@) {
3910             eval { qemu_drivedel($vmid, $deviceid); };
3911             warn $@ if $@;
3912             die $err;
3913         }
3914
3915     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3916
3917         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3918
3919         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3920         my $use_old_bios_files = undef;
3921         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3922
3923         my $netdevicefull = print_netdevice_full(
3924             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3925         qemu_deviceadd($vmid, $netdevicefull);
3926         eval {
3927             qemu_deviceaddverify($vmid, $deviceid);
3928             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3929         };
3930         if (my $err = $@) {
3931             eval { qemu_netdevdel($vmid, $deviceid); };
3932             warn $@ if $@;
3933             die $err;
3934         }
3935
3936     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3937
3938         my $bridgeid = $2;
3939         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3940         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3941
3942         qemu_deviceadd($vmid, $devicefull);
3943         qemu_deviceaddverify($vmid, $deviceid);
3944
3945     } else {
3946         die "can't hotplug device '$deviceid'\n";
3947     }
3948
3949     return 1;
3950 }
3951
3952 # fixme: this should raise exceptions on error!
3953 sub vm_deviceunplug {
3954     my ($vmid, $conf, $deviceid) = @_;
3955
3956     my $devices_list = vm_devices_list($vmid);
3957     return 1 if !defined($devices_list->{$deviceid});
3958
3959     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3960     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3961
3962     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3963
3964         qemu_devicedel($vmid, $deviceid);
3965
3966     } elsif ($deviceid =~ m/^usb\d+$/) {
3967
3968         die "usb hotplug currently not reliable\n";
3969         # when unplugging usb devices this way, there may be remaining usb
3970         # controllers/hubs so we disable it for now
3971         #qemu_devicedel($vmid, $deviceid);
3972         #qemu_devicedelverify($vmid, $deviceid);
3973
3974     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3975
3976         qemu_devicedel($vmid, $deviceid);
3977         qemu_devicedelverify($vmid, $deviceid);
3978         qemu_drivedel($vmid, $deviceid);
3979         qemu_iothread_del($conf, $vmid, $deviceid);
3980
3981     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3982
3983         qemu_devicedel($vmid, $deviceid);
3984         qemu_devicedelverify($vmid, $deviceid);
3985         qemu_iothread_del($conf, $vmid, $deviceid);
3986
3987     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3988
3989         qemu_devicedel($vmid, $deviceid);
3990         qemu_drivedel($vmid, $deviceid);
3991         qemu_deletescsihw($conf, $vmid, $deviceid);
3992
3993     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3994
3995         qemu_devicedel($vmid, $deviceid);
3996         qemu_devicedelverify($vmid, $deviceid);
3997         qemu_netdevdel($vmid, $deviceid);
3998
3999     } else {
4000         die "can't unplug device '$deviceid'\n";
4001     }
4002
4003     return 1;
4004 }
4005
4006 sub qemu_deviceadd {
4007     my ($vmid, $devicefull) = @_;
4008
4009     $devicefull = "driver=".$devicefull;
4010     my %options =  split(/[=,]/, $devicefull);
4011
4012     mon_cmd($vmid, "device_add" , %options);
4013 }
4014
4015 sub qemu_devicedel {
4016     my ($vmid, $deviceid) = @_;
4017
4018     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4019 }
4020
4021 sub qemu_iothread_add {
4022     my($vmid, $deviceid, $device) = @_;
4023
4024     if ($device->{iothread}) {
4025         my $iothreads = vm_iothreads_list($vmid);
4026         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4027     }
4028 }
4029
4030 sub qemu_iothread_del {
4031     my($conf, $vmid, $deviceid) = @_;
4032
4033     my $confid = $deviceid;
4034     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4035         $confid = 'scsi' . $1;
4036     }
4037     my $device = parse_drive($confid, $conf->{$confid});
4038     if ($device->{iothread}) {
4039         my $iothreads = vm_iothreads_list($vmid);
4040         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4041     }
4042 }
4043
4044 sub qemu_objectadd {
4045     my($vmid, $objectid, $qomtype) = @_;
4046
4047     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4048
4049     return 1;
4050 }
4051
4052 sub qemu_objectdel {
4053     my($vmid, $objectid) = @_;
4054
4055     mon_cmd($vmid, "object-del", id => $objectid);
4056
4057     return 1;
4058 }
4059
4060 sub qemu_driveadd {
4061     my ($storecfg, $vmid, $device) = @_;
4062
4063     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4064     $drive =~ s/\\/\\\\/g;
4065     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4066
4067     # If the command succeeds qemu prints: "OK"
4068     return 1 if $ret =~ m/OK/s;
4069
4070     die "adding drive failed: $ret\n";
4071 }
4072
4073 sub qemu_drivedel {
4074     my($vmid, $deviceid) = @_;
4075
4076     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4077     $ret =~ s/^\s+//;
4078
4079     return 1 if $ret eq "";
4080
4081     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4082     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4083
4084     die "deleting drive $deviceid failed : $ret\n";
4085 }
4086
4087 sub qemu_deviceaddverify {
4088     my ($vmid, $deviceid) = @_;
4089
4090     for (my $i = 0; $i <= 5; $i++) {
4091          my $devices_list = vm_devices_list($vmid);
4092          return 1 if defined($devices_list->{$deviceid});
4093          sleep 1;
4094     }
4095
4096     die "error on hotplug device '$deviceid'\n";
4097 }
4098
4099
4100 sub qemu_devicedelverify {
4101     my ($vmid, $deviceid) = @_;
4102
4103     # need to verify that the device is correctly removed as device_del
4104     # is async and empty return is not reliable
4105
4106     for (my $i = 0; $i <= 5; $i++) {
4107          my $devices_list = vm_devices_list($vmid);
4108          return 1 if !defined($devices_list->{$deviceid});
4109          sleep 1;
4110     }
4111
4112     die "error on hot-unplugging device '$deviceid'\n";
4113 }
4114
4115 sub qemu_findorcreatescsihw {
4116     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4117
4118     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4119
4120     my $scsihwid="$controller_prefix$controller";
4121     my $devices_list = vm_devices_list($vmid);
4122
4123     if(!defined($devices_list->{$scsihwid})) {
4124         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4125     }
4126
4127     return 1;
4128 }
4129
4130 sub qemu_deletescsihw {
4131     my ($conf, $vmid, $opt) = @_;
4132
4133     my $device = parse_drive($opt, $conf->{$opt});
4134
4135     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4136         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4137         return 1;
4138     }
4139
4140     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4141
4142     my $devices_list = vm_devices_list($vmid);
4143     foreach my $opt (keys %{$devices_list}) {
4144         if (is_valid_drivename($opt)) {
4145             my $drive = parse_drive($opt, $conf->{$opt});
4146             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4147                 return 1;
4148             }
4149         }
4150     }
4151
4152     my $scsihwid="scsihw$controller";
4153
4154     vm_deviceunplug($vmid, $conf, $scsihwid);
4155
4156     return 1;
4157 }
4158
4159 sub qemu_add_pci_bridge {
4160     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4161
4162     my $bridges = {};
4163
4164     my $bridgeid;
4165
4166     print_pci_addr($device, $bridges, $arch, $machine_type);
4167
4168     while (my ($k, $v) = each %$bridges) {
4169         $bridgeid = $k;
4170     }
4171     return 1 if !defined($bridgeid) || $bridgeid < 1;
4172
4173     my $bridge = "pci.$bridgeid";
4174     my $devices_list = vm_devices_list($vmid);
4175
4176     if (!defined($devices_list->{$bridge})) {
4177         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4178     }
4179
4180     return 1;
4181 }
4182
4183 sub qemu_set_link_status {
4184     my ($vmid, $device, $up) = @_;
4185
4186     mon_cmd($vmid, "set_link", name => $device,
4187                up => $up ? JSON::true : JSON::false);
4188 }
4189
4190 sub qemu_netdevadd {
4191     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4192
4193     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4194     my %options =  split(/[=,]/, $netdev);
4195
4196     if (defined(my $vhost = $options{vhost})) {
4197         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4198     }
4199
4200     if (defined(my $queues = $options{queues})) {
4201         $options{queues} = $queues + 0;
4202     }
4203
4204     mon_cmd($vmid, "netdev_add",  %options);
4205     return 1;
4206 }
4207
4208 sub qemu_netdevdel {
4209     my ($vmid, $deviceid) = @_;
4210
4211     mon_cmd($vmid, "netdev_del", id => $deviceid);
4212 }
4213
4214 sub qemu_usb_hotplug {
4215     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4216
4217     return if !$device;
4218
4219     # remove the old one first
4220     vm_deviceunplug($vmid, $conf, $deviceid);
4221
4222     # check if xhci controller is necessary and available
4223     if ($device->{usb3}) {
4224
4225         my $devicelist = vm_devices_list($vmid);
4226
4227         if (!$devicelist->{xhci}) {
4228             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4229             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4230         }
4231     }
4232     my $d = parse_usb_device($device->{host});
4233     $d->{usb3} = $device->{usb3};
4234
4235     # add the new one
4236     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4237 }
4238
4239 sub qemu_cpu_hotplug {
4240     my ($vmid, $conf, $vcpus) = @_;
4241
4242     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4243
4244     my $sockets = 1;
4245     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4246     $sockets = $conf->{sockets} if  $conf->{sockets};
4247     my $cores = $conf->{cores} || 1;
4248     my $maxcpus = $sockets * $cores;
4249
4250     $vcpus = $maxcpus if !$vcpus;
4251
4252     die "you can't add more vcpus than maxcpus\n"
4253         if $vcpus > $maxcpus;
4254
4255     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4256
4257     if ($vcpus < $currentvcpus) {
4258
4259         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4260
4261             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4262                 qemu_devicedel($vmid, "cpu$i");
4263                 my $retry = 0;
4264                 my $currentrunningvcpus = undef;
4265                 while (1) {
4266                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4267                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4268                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4269                     $retry++;
4270                     sleep 1;
4271                 }
4272                 #update conf after each succesfull cpu unplug
4273                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4274                 PVE::QemuConfig->write_config($vmid, $conf);
4275             }
4276         } else {
4277             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4278         }
4279
4280         return;
4281     }
4282
4283     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4284     die "vcpus in running vm does not match its configuration\n"
4285         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4286
4287     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4288
4289         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4290             my $cpustr = print_cpu_device($conf, $i);
4291             qemu_deviceadd($vmid, $cpustr);
4292
4293             my $retry = 0;
4294             my $currentrunningvcpus = undef;
4295             while (1) {
4296                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4297                 last if scalar(@{$currentrunningvcpus}) == $i;
4298                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4299                 sleep 1;
4300                 $retry++;
4301             }
4302             #update conf after each succesfull cpu hotplug
4303             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4304             PVE::QemuConfig->write_config($vmid, $conf);
4305         }
4306     } else {
4307
4308         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4309             mon_cmd($vmid, "cpu-add", id => int($i));
4310         }
4311     }
4312 }
4313
4314 sub qemu_block_set_io_throttle {
4315     my ($vmid, $deviceid,
4316         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4317         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4318         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4319         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4320
4321     return if !check_running($vmid) ;
4322
4323     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4324         bps => int($bps),
4325         bps_rd => int($bps_rd),
4326         bps_wr => int($bps_wr),
4327         iops => int($iops),
4328         iops_rd => int($iops_rd),
4329         iops_wr => int($iops_wr),
4330         bps_max => int($bps_max),
4331         bps_rd_max => int($bps_rd_max),
4332         bps_wr_max => int($bps_wr_max),
4333         iops_max => int($iops_max),
4334         iops_rd_max => int($iops_rd_max),
4335         iops_wr_max => int($iops_wr_max),
4336         bps_max_length => int($bps_max_length),
4337         bps_rd_max_length => int($bps_rd_max_length),
4338         bps_wr_max_length => int($bps_wr_max_length),
4339         iops_max_length => int($iops_max_length),
4340         iops_rd_max_length => int($iops_rd_max_length),
4341         iops_wr_max_length => int($iops_wr_max_length),
4342     );
4343
4344 }
4345
4346 sub qemu_block_resize {
4347     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4348
4349     my $running = check_running($vmid);
4350
4351     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4352
4353     return if !$running;
4354
4355     my $padding = (1024 - $size % 1024) % 1024;
4356     $size = $size + $padding;
4357
4358     mon_cmd(
4359         $vmid,
4360         "block_resize",
4361         device => $deviceid,
4362         size => int($size),
4363         timeout => 60,
4364     );
4365 }
4366
4367 sub qemu_volume_snapshot {
4368     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4369
4370     my $running = check_running($vmid);
4371
4372     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4373         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4374     } else {
4375         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4376     }
4377 }
4378
4379 sub qemu_volume_snapshot_delete {
4380     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4381
4382     my $running = check_running($vmid);
4383
4384     if($running) {
4385
4386         $running = undef;
4387         my $conf = PVE::QemuConfig->load_config($vmid);
4388         PVE::QemuConfig->foreach_volume($conf, sub {
4389             my ($ds, $drive) = @_;
4390             $running = 1 if $drive->{file} eq $volid;
4391         });
4392     }
4393
4394     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4395         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4396     } else {
4397         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4398     }
4399 }
4400
4401 sub set_migration_caps {
4402     my ($vmid, $savevm) = @_;
4403
4404     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4405
4406     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4407     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4408
4409     my $cap_ref = [];
4410
4411     my $enabled_cap = {
4412         "auto-converge" => 1,
4413         "xbzrle" => 1,
4414         "x-rdma-pin-all" => 0,
4415         "zero-blocks" => 0,
4416         "compress" => 0,
4417         "dirty-bitmaps" => $dirty_bitmaps,
4418     };
4419
4420     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4421
4422     for my $supported_capability (@$supported_capabilities) {
4423         push @$cap_ref, {
4424             capability => $supported_capability->{capability},
4425             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4426         };
4427     }
4428
4429     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4430 }
4431
4432 sub foreach_volid {
4433     my ($conf, $func, @param) = @_;
4434
4435     my $volhash = {};
4436
4437     my $test_volid = sub {
4438         my ($key, $drive, $snapname) = @_;
4439
4440         my $volid = $drive->{file};
4441         return if !$volid;
4442
4443         $volhash->{$volid}->{cdrom} //= 1;
4444         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4445
4446         my $replicate = $drive->{replicate} // 1;
4447         $volhash->{$volid}->{replicate} //= 0;
4448         $volhash->{$volid}->{replicate} = 1 if $replicate;
4449
4450         $volhash->{$volid}->{shared} //= 0;
4451         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4452
4453         $volhash->{$volid}->{referenced_in_config} //= 0;
4454         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4455
4456         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4457             if defined($snapname);
4458
4459         my $size = $drive->{size};
4460         $volhash->{$volid}->{size} //= $size if $size;
4461
4462         $volhash->{$volid}->{is_vmstate} //= 0;
4463         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4464
4465         $volhash->{$volid}->{is_unused} //= 0;
4466         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4467
4468         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4469     };
4470
4471     my $include_opts = {
4472         extra_keys => ['vmstate'],
4473         include_unused => 1,
4474     };
4475
4476     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4477     foreach my $snapname (keys %{$conf->{snapshots}}) {
4478         my $snap = $conf->{snapshots}->{$snapname};
4479         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4480     }
4481
4482     foreach my $volid (keys %$volhash) {
4483         &$func($volid, $volhash->{$volid}, @param);
4484     }
4485 }
4486
4487 my $fast_plug_option = {
4488     'lock' => 1,
4489     'name' => 1,
4490     'onboot' => 1,
4491     'shares' => 1,
4492     'startup' => 1,
4493     'description' => 1,
4494     'protection' => 1,
4495     'vmstatestorage' => 1,
4496     'hookscript' => 1,
4497     'tags' => 1,
4498 };
4499
4500 # hotplug changes in [PENDING]
4501 # $selection hash can be used to only apply specified options, for
4502 # example: { cores => 1 } (only apply changed 'cores')
4503 # $errors ref is used to return error messages
4504 sub vmconfig_hotplug_pending {
4505     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4506
4507     my $defaults = load_defaults();
4508     my $arch = get_vm_arch($conf);
4509     my $machine_type = get_vm_machine($conf, undef, $arch);
4510
4511     # commit values which do not have any impact on running VM first
4512     # Note: those option cannot raise errors, we we do not care about
4513     # $selection and always apply them.
4514
4515     my $add_error = sub {
4516         my ($opt, $msg) = @_;
4517         $errors->{$opt} = "hotplug problem - $msg";
4518     };
4519
4520     my $changes = 0;
4521     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4522         if ($fast_plug_option->{$opt}) {
4523             $conf->{$opt} = $conf->{pending}->{$opt};
4524             delete $conf->{pending}->{$opt};
4525             $changes = 1;
4526         }
4527     }
4528
4529     if ($changes) {
4530         PVE::QemuConfig->write_config($vmid, $conf);
4531     }
4532
4533     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4534
4535     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4536     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4537     foreach my $opt (sort keys %$pending_delete_hash) {
4538         next if $selection && !$selection->{$opt};
4539         my $force = $pending_delete_hash->{$opt}->{force};
4540         eval {
4541             if ($opt eq 'hotplug') {
4542                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4543             } elsif ($opt eq 'tablet') {
4544                 die "skip\n" if !$hotplug_features->{usb};
4545                 if ($defaults->{tablet}) {
4546                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4547                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4548                         if $arch eq 'aarch64';
4549                 } else {
4550                     vm_deviceunplug($vmid, $conf, 'tablet');
4551                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4552                 }
4553             } elsif ($opt =~ m/^usb\d+/) {
4554                 die "skip\n";
4555                 # since we cannot reliably hot unplug usb devices we are disabling it
4556                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4557                 #vm_deviceunplug($vmid, $conf, $opt);
4558             } elsif ($opt eq 'vcpus') {
4559                 die "skip\n" if !$hotplug_features->{cpu};
4560                 qemu_cpu_hotplug($vmid, $conf, undef);
4561             } elsif ($opt eq 'balloon') {
4562                 # enable balloon device is not hotpluggable
4563                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4564                 # here we reset the ballooning value to memory
4565                 my $balloon = $conf->{memory} || $defaults->{memory};
4566                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4567             } elsif ($fast_plug_option->{$opt}) {
4568                 # do nothing
4569             } elsif ($opt =~ m/^net(\d+)$/) {
4570                 die "skip\n" if !$hotplug_features->{network};
4571                 vm_deviceunplug($vmid, $conf, $opt);
4572             } elsif (is_valid_drivename($opt)) {
4573                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4574                 vm_deviceunplug($vmid, $conf, $opt);
4575                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4576             } elsif ($opt =~ m/^memory$/) {
4577                 die "skip\n" if !$hotplug_features->{memory};
4578                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4579             } elsif ($opt eq 'cpuunits') {
4580                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4581             } elsif ($opt eq 'cpulimit') {
4582                 $cgroup->change_cpu_quota(-1, 100000);
4583             } else {
4584                 die "skip\n";
4585             }
4586         };
4587         if (my $err = $@) {
4588             &$add_error($opt, $err) if $err ne "skip\n";
4589         } else {
4590             delete $conf->{$opt};
4591             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4592         }
4593     }
4594
4595     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4596     $apply_pending_cloudinit = sub {
4597         return if $apply_pending_cloudinit_done; # once is enough
4598         $apply_pending_cloudinit_done = 1; # once is enough
4599
4600         my ($key, $value) = @_;
4601
4602         my @cloudinit_opts = keys %$confdesc_cloudinit;
4603         foreach my $opt (keys %{$conf->{pending}}) {
4604             next if !grep { $_ eq $opt } @cloudinit_opts;
4605             $conf->{$opt} = delete $conf->{pending}->{$opt};
4606         }
4607
4608         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4609         foreach my $opt (sort keys %$pending_delete_hash) {
4610             next if !grep { $_ eq $opt } @cloudinit_opts;
4611             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4612             delete $conf->{$opt};
4613         }
4614
4615         my $new_conf = { %$conf };
4616         $new_conf->{$key} = $value;
4617         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4618     };
4619
4620     foreach my $opt (keys %{$conf->{pending}}) {
4621         next if $selection && !$selection->{$opt};
4622         my $value = $conf->{pending}->{$opt};
4623         eval {
4624             if ($opt eq 'hotplug') {
4625                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4626             } elsif ($opt eq 'tablet') {
4627                 die "skip\n" if !$hotplug_features->{usb};
4628                 if ($value == 1) {
4629                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4630                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4631                         if $arch eq 'aarch64';
4632                 } elsif ($value == 0) {
4633                     vm_deviceunplug($vmid, $conf, 'tablet');
4634                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4635                 }
4636             } elsif ($opt =~ m/^usb\d+$/) {
4637                 die "skip\n";
4638                 # since we cannot reliably hot unplug usb devices we disable it for now
4639                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4640                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4641                 #die "skip\n" if !$d;
4642                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4643             } elsif ($opt eq 'vcpus') {
4644                 die "skip\n" if !$hotplug_features->{cpu};
4645                 qemu_cpu_hotplug($vmid, $conf, $value);
4646             } elsif ($opt eq 'balloon') {
4647                 # enable/disable balloning device is not hotpluggable
4648                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4649                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4650                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4651
4652                 # allow manual ballooning if shares is set to zero
4653                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4654                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4655                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4656                 }
4657             } elsif ($opt =~ m/^net(\d+)$/) {
4658                 # some changes can be done without hotplug
4659                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4660                                     $vmid, $opt, $value, $arch, $machine_type);
4661             } elsif (is_valid_drivename($opt)) {
4662                 die "skip\n" if $opt eq 'efidisk0';
4663                 # some changes can be done without hotplug
4664                 my $drive = parse_drive($opt, $value);
4665                 if (drive_is_cloudinit($drive)) {
4666                     &$apply_pending_cloudinit($opt, $value);
4667                 }
4668                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4669                                      $vmid, $opt, $value, $arch, $machine_type);
4670             } elsif ($opt =~ m/^memory$/) { #dimms
4671                 die "skip\n" if !$hotplug_features->{memory};
4672                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4673             } elsif ($opt eq 'cpuunits') {
4674                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4675             } elsif ($opt eq 'cpulimit') {
4676                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4677                 $cgroup->change_cpu_quota($cpulimit, 100000);
4678             } else {
4679                 die "skip\n";  # skip non-hot-pluggable options
4680             }
4681         };
4682         if (my $err = $@) {
4683             &$add_error($opt, $err) if $err ne "skip\n";
4684         } else {
4685             $conf->{$opt} = $value;
4686             delete $conf->{pending}->{$opt};
4687         }
4688     }
4689
4690     PVE::QemuConfig->write_config($vmid, $conf);
4691 }
4692
4693 sub try_deallocate_drive {
4694     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4695
4696     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4697         my $volid = $drive->{file};
4698         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4699             my $sid = PVE::Storage::parse_volume_id($volid);
4700             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4701
4702             # check if the disk is really unused
4703             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4704                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4705             PVE::Storage::vdisk_free($storecfg, $volid);
4706             return 1;
4707         } else {
4708             # If vm is not owner of this disk remove from config
4709             return 1;
4710         }
4711     }
4712
4713     return;
4714 }
4715
4716 sub vmconfig_delete_or_detach_drive {
4717     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4718
4719     my $drive = parse_drive($opt, $conf->{$opt});
4720
4721     my $rpcenv = PVE::RPCEnvironment::get();
4722     my $authuser = $rpcenv->get_user();
4723
4724     if ($force) {
4725         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4726         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4727     } else {
4728         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4729     }
4730 }
4731
4732
4733
4734 sub vmconfig_apply_pending {
4735     my ($vmid, $conf, $storecfg, $errors) = @_;
4736
4737     my $add_apply_error = sub {
4738         my ($opt, $msg) = @_;
4739         my $err_msg = "unable to apply pending change $opt : $msg";
4740         $errors->{$opt} = $err_msg;
4741         warn $err_msg;
4742     };
4743
4744     # cold plug
4745
4746     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4747     foreach my $opt (sort keys %$pending_delete_hash) {
4748         my $force = $pending_delete_hash->{$opt}->{force};
4749         eval {
4750             if ($opt =~ m/^unused/) {
4751                 die "internal error";
4752             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4753                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4754             }
4755         };
4756         if (my $err = $@) {
4757             $add_apply_error->($opt, $err);
4758         } else {
4759             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4760             delete $conf->{$opt};
4761         }
4762     }
4763
4764     PVE::QemuConfig->cleanup_pending($conf);
4765
4766     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4767         next if $opt eq 'delete'; # just to be sure
4768         eval {
4769             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4770                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4771             }
4772         };
4773         if (my $err = $@) {
4774             $add_apply_error->($opt, $err);
4775         } else {
4776             $conf->{$opt} = delete $conf->{pending}->{$opt};
4777         }
4778     }
4779
4780     # write all changes at once to avoid unnecessary i/o
4781     PVE::QemuConfig->write_config($vmid, $conf);
4782 }
4783
4784 sub vmconfig_update_net {
4785     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4786
4787     my $newnet = parse_net($value);
4788
4789     if ($conf->{$opt}) {
4790         my $oldnet = parse_net($conf->{$opt});
4791
4792         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4793             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4794             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4795             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4796
4797             # for non online change, we try to hot-unplug
4798             die "skip\n" if !$hotplug;
4799             vm_deviceunplug($vmid, $conf, $opt);
4800         } else {
4801
4802             die "internal error" if $opt !~ m/net(\d+)/;
4803             my $iface = "tap${vmid}i$1";
4804
4805             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4806                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4807                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4808                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4809                 PVE::Network::tap_unplug($iface);
4810
4811                 if ($have_sdn) {
4812                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4813                 } else {
4814                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4815                 }
4816             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4817                 # Rate can be applied on its own but any change above needs to
4818                 # include the rate in tap_plug since OVS resets everything.
4819                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4820             }
4821
4822             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4823                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4824             }
4825
4826             return 1;
4827         }
4828     }
4829
4830     if ($hotplug) {
4831         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4832     } else {
4833         die "skip\n";
4834     }
4835 }
4836
4837 sub vmconfig_update_disk {
4838     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4839
4840     my $drive = parse_drive($opt, $value);
4841
4842     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4843         my $media = $drive->{media} || 'disk';
4844         my $oldmedia = $old_drive->{media} || 'disk';
4845         die "unable to change media type\n" if $media ne $oldmedia;
4846
4847         if (!drive_is_cdrom($old_drive)) {
4848
4849             if ($drive->{file} ne $old_drive->{file}) {
4850
4851                 die "skip\n" if !$hotplug;
4852
4853                 # unplug and register as unused
4854                 vm_deviceunplug($vmid, $conf, $opt);
4855                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4856
4857             } else {
4858                 # update existing disk
4859
4860                 # skip non hotpluggable value
4861                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4862                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4863                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4864                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4865                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4866                     die "skip\n";
4867                 }
4868
4869                 # apply throttle
4870                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4871                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4872                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4873                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4874                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4875                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4876                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4877                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4878                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4879                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4880                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4881                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4882                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4883                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4884                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4885                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4886                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4887                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4888
4889                     qemu_block_set_io_throttle(
4890                         $vmid,"drive-$opt",
4891                         ($drive->{mbps} || 0)*1024*1024,
4892                         ($drive->{mbps_rd} || 0)*1024*1024,
4893                         ($drive->{mbps_wr} || 0)*1024*1024,
4894                          $drive->{iops} || 0,
4895                          $drive->{iops_rd} || 0,
4896                          $drive->{iops_wr} || 0,
4897                         ($drive->{mbps_max} || 0)*1024*1024,
4898                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4899                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4900                          $drive->{iops_max} || 0,
4901                          $drive->{iops_rd_max} || 0,
4902                          $drive->{iops_wr_max} || 0,
4903                          $drive->{bps_max_length} || 1,
4904                          $drive->{bps_rd_max_length} || 1,
4905                          $drive->{bps_wr_max_length} || 1,
4906                          $drive->{iops_max_length} || 1,
4907                          $drive->{iops_rd_max_length} || 1,
4908                          $drive->{iops_wr_max_length} || 1,
4909                     );
4910
4911                 }
4912
4913                 return 1;
4914             }
4915
4916         } else { # cdrom
4917
4918             if ($drive->{file} eq 'none') {
4919                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4920                 if (drive_is_cloudinit($old_drive)) {
4921                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4922                 }
4923             } else {
4924                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4925
4926                 # force eject if locked
4927                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4928
4929                 if ($path) {
4930                     mon_cmd($vmid, "blockdev-change-medium",
4931                         id => "$opt", filename => "$path");
4932                 }
4933             }
4934
4935             return 1;
4936         }
4937     }
4938
4939     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4940     # hotplug new disks
4941     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4942     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4943 }
4944
4945 # called in locked context by incoming migration
4946 sub vm_migrate_get_nbd_disks {
4947     my ($storecfg, $conf, $replicated_volumes) = @_;
4948
4949     my $local_volumes = {};
4950     PVE::QemuConfig->foreach_volume($conf, sub {
4951         my ($ds, $drive) = @_;
4952
4953         return if drive_is_cdrom($drive);
4954
4955         my $volid = $drive->{file};
4956
4957         return if !$volid;
4958
4959         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4960
4961         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4962         return if $scfg->{shared};
4963
4964         # replicated disks re-use existing state via bitmap
4965         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4966         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4967     });
4968     return $local_volumes;
4969 }
4970
4971 # called in locked context by incoming migration
4972 sub vm_migrate_alloc_nbd_disks {
4973     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4974
4975     my $format = undef;
4976
4977     my $nbd = {};
4978     foreach my $opt (sort keys %$source_volumes) {
4979         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4980
4981         if ($use_existing) {
4982             $nbd->{$opt}->{drivestr} = print_drive($drive);
4983             $nbd->{$opt}->{volid} = $volid;
4984             $nbd->{$opt}->{replicated} = 1;
4985             next;
4986         }
4987
4988         # If a remote storage is specified and the format of the original
4989         # volume is not available there, fall back to the default format.
4990         # Otherwise use the same format as the original.
4991         if (!$storagemap->{identity}) {
4992             $storeid = map_storage($storagemap, $storeid);
4993             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4994             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4995             my $fileFormat = qemu_img_format($scfg, $volname);
4996             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4997         } else {
4998             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4999             $format = qemu_img_format($scfg, $volname);
5000         }
5001
5002         my $size = $drive->{size} / 1024;
5003         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5004         my $newdrive = $drive;
5005         $newdrive->{format} = $format;
5006         $newdrive->{file} = $newvolid;
5007         my $drivestr = print_drive($newdrive);
5008         $nbd->{$opt}->{drivestr} = $drivestr;
5009         $nbd->{$opt}->{volid} = $newvolid;
5010     }
5011
5012     return $nbd;
5013 }
5014
5015 # see vm_start_nolock for parameters, additionally:
5016 # migrate_opts:
5017 #   storagemap = parsed storage map for allocating NBD disks
5018 sub vm_start {
5019     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5020
5021     return PVE::QemuConfig->lock_config($vmid, sub {
5022         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5023
5024         die "you can't start a vm if it's a template\n"
5025             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5026
5027         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5028         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5029
5030         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5031
5032         if ($has_backup_lock && $running) {
5033             # a backup is currently running, attempt to start the guest in the
5034             # existing QEMU instance
5035             return vm_resume($vmid);
5036         }
5037
5038         PVE::QemuConfig->check_lock($conf)
5039             if !($params->{skiplock} || $has_suspended_lock);
5040
5041         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5042
5043         die "VM $vmid already running\n" if $running;
5044
5045         if (my $storagemap = $migrate_opts->{storagemap}) {
5046             my $replicated = $migrate_opts->{replicated_volumes};
5047             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5048             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5049
5050             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5051                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5052             }
5053         }
5054
5055         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5056     });
5057 }
5058
5059
5060 # params:
5061 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5062 #   skiplock => 0/1, skip checking for config lock
5063 #   skiptemplate => 0/1, skip checking whether VM is template
5064 #   forcemachine => to force Qemu machine (rollback/migration)
5065 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5066 #   timeout => in seconds
5067 #   paused => start VM in paused state (backup)
5068 #   resume => resume from hibernation
5069 #   pbs-backing => {
5070 #      sata0 => {
5071 #         repository
5072 #         snapshot
5073 #         keyfile
5074 #         archive
5075 #      },
5076 #      virtio2 => ...
5077 #   }
5078 # migrate_opts:
5079 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5080 #   migratedfrom => source node
5081 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5082 #   network => CIDR of migration network
5083 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5084 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5085 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5086 sub vm_start_nolock {
5087     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5088
5089     my $statefile = $params->{statefile};
5090     my $resume = $params->{resume};
5091
5092     my $migratedfrom = $migrate_opts->{migratedfrom};
5093     my $migration_type = $migrate_opts->{type};
5094
5095     my $res = {};
5096
5097     # clean up leftover reboot request files
5098     eval { clear_reboot_request($vmid); };
5099     warn $@ if $@;
5100
5101     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5102         vmconfig_apply_pending($vmid, $conf, $storecfg);
5103         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5104     }
5105
5106     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5107
5108     my $defaults = load_defaults();
5109
5110     # set environment variable useful inside network script
5111     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5112
5113     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5114
5115     my $forcemachine = $params->{forcemachine};
5116     my $forcecpu = $params->{forcecpu};
5117     if ($resume) {
5118         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5119         $forcemachine = $conf->{runningmachine};
5120         $forcecpu = $conf->{runningcpu};
5121         print "Resuming suspended VM\n";
5122     }
5123
5124     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5125         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5126
5127     my $migration_ip;
5128     my $get_migration_ip = sub {
5129         my ($nodename) = @_;
5130
5131         return $migration_ip if defined($migration_ip);
5132
5133         my $cidr = $migrate_opts->{network};
5134
5135         if (!defined($cidr)) {
5136             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5137             $cidr = $dc_conf->{migration}->{network};
5138         }
5139
5140         if (defined($cidr)) {
5141             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5142
5143             die "could not get IP: no address configured on local " .
5144                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5145
5146             die "could not get IP: multiple addresses configured on local " .
5147                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5148
5149             $migration_ip = @$ips[0];
5150         }
5151
5152         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5153             if !defined($migration_ip);
5154
5155         return $migration_ip;
5156     };
5157
5158     my $migrate_uri;
5159     if ($statefile) {
5160         if ($statefile eq 'tcp') {
5161             my $localip = "localhost";
5162             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5163             my $nodename = nodename();
5164
5165             if (!defined($migration_type)) {
5166                 if (defined($datacenterconf->{migration}->{type})) {
5167                     $migration_type = $datacenterconf->{migration}->{type};
5168                 } else {
5169                     $migration_type = 'secure';
5170                 }
5171             }
5172
5173             if ($migration_type eq 'insecure') {
5174                 $localip = $get_migration_ip->($nodename);
5175                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5176             }
5177
5178             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5179             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5180             $migrate_uri = "tcp:${localip}:${migrate_port}";
5181             push @$cmd, '-incoming', $migrate_uri;
5182             push @$cmd, '-S';
5183
5184         } elsif ($statefile eq 'unix') {
5185             # should be default for secure migrations as a ssh TCP forward
5186             # tunnel is not deterministic reliable ready and fails regurarly
5187             # to set up in time, so use UNIX socket forwards
5188             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5189             unlink $socket_addr;
5190
5191             $migrate_uri = "unix:$socket_addr";
5192
5193             push @$cmd, '-incoming', $migrate_uri;
5194             push @$cmd, '-S';
5195
5196         } elsif (-e $statefile) {
5197             push @$cmd, '-loadstate', $statefile;
5198         } else {
5199             my $statepath = PVE::Storage::path($storecfg, $statefile);
5200             push @$vollist, $statefile;
5201             push @$cmd, '-loadstate', $statepath;
5202         }
5203     } elsif ($params->{paused}) {
5204         push @$cmd, '-S';
5205     }
5206
5207     # host pci devices
5208     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5209       my $d = parse_hostpci($conf->{"hostpci$i"});
5210       next if !$d;
5211       my $pcidevices = $d->{pciid};
5212       foreach my $pcidevice (@$pcidevices) {
5213             my $pciid = $pcidevice->{id};
5214
5215             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5216             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5217             die "no pci device info for device '$pciid'\n" if !$info;
5218
5219             if ($d->{mdev}) {
5220                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5221                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5222             } else {
5223                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5224                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5225                 die "can't reset PCI device '$pciid'\n"
5226                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5227             }
5228       }
5229     }
5230
5231     PVE::Storage::activate_volumes($storecfg, $vollist);
5232
5233     eval {
5234         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5235             outfunc => sub {}, errfunc => sub {});
5236     };
5237     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5238     # timeout should be more than enough here...
5239     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5240
5241     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5242                                               : $defaults->{cpuunits};
5243
5244     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5245     my %run_params = (
5246         timeout => $statefile ? undef : $start_timeout,
5247         umask => 0077,
5248         noerr => 1,
5249     );
5250
5251     # when migrating, prefix QEMU output so other side can pick up any
5252     # errors that might occur and show the user
5253     if ($migratedfrom) {
5254         $run_params{quiet} = 1;
5255         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5256     }
5257
5258     my %properties = (
5259         Slice => 'qemu.slice',
5260         KillMode => 'none'
5261     );
5262
5263     if (PVE::CGroup::cgroup_mode() == 2) {
5264         $properties{CPUWeight} = $cpuunits;
5265     } else {
5266         $properties{CPUShares} = $cpuunits;
5267     }
5268
5269     if (my $cpulimit = $conf->{cpulimit}) {
5270         $properties{CPUQuota} = int($cpulimit * 100);
5271     }
5272     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5273
5274     my $run_qemu = sub {
5275         PVE::Tools::run_fork sub {
5276             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5277
5278             my $exitcode = run_command($cmd, %run_params);
5279             die "QEMU exited with code $exitcode\n" if $exitcode;
5280         };
5281     };
5282
5283     if ($conf->{hugepages}) {
5284
5285         my $code = sub {
5286             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5287             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5288
5289             PVE::QemuServer::Memory::hugepages_mount();
5290             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5291
5292             eval { $run_qemu->() };
5293             if (my $err = $@) {
5294                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5295                     if !$conf->{keephugepages};
5296                 die $err;
5297             }
5298
5299             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5300                 if !$conf->{keephugepages};
5301         };
5302         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5303
5304     } else {
5305         eval { $run_qemu->() };
5306     }
5307
5308     if (my $err = $@) {
5309         # deactivate volumes if start fails
5310         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5311         die "start failed: $err";
5312     }
5313
5314     print "migration listens on $migrate_uri\n" if $migrate_uri;
5315     $res->{migrate_uri} = $migrate_uri;
5316
5317     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5318         eval { mon_cmd($vmid, "cont"); };
5319         warn $@ if $@;
5320     }
5321
5322     #start nbd server for storage migration
5323     if (my $nbd = $migrate_opts->{nbd}) {
5324         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5325
5326         my $migrate_storage_uri;
5327         # nbd_protocol_version > 0 for unix socket support
5328         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5329             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5330             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5331             $migrate_storage_uri = "nbd:unix:$socket_path";
5332         } else {
5333             my $nodename = nodename();
5334             my $localip = $get_migration_ip->($nodename);
5335             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5336             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5337
5338             mon_cmd($vmid, "nbd-server-start", addr => {
5339                 type => 'inet',
5340                 data => {
5341                     host => "${localip}",
5342                     port => "${storage_migrate_port}",
5343                 },
5344             });
5345             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5346             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5347         }
5348
5349         $res->{migrate_storage_uri} = $migrate_storage_uri;
5350
5351         foreach my $opt (sort keys %$nbd) {
5352             my $drivestr = $nbd->{$opt}->{drivestr};
5353             my $volid = $nbd->{$opt}->{volid};
5354             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5355             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5356             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5357             print "re-using replicated volume: $opt - $volid\n"
5358                 if $nbd->{$opt}->{replicated};
5359
5360             $res->{drives}->{$opt} = $nbd->{$opt};
5361             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5362         }
5363     }
5364
5365     if ($migratedfrom) {
5366         eval {
5367             set_migration_caps($vmid);
5368         };
5369         warn $@ if $@;
5370
5371         if ($spice_port) {
5372             print "spice listens on port $spice_port\n";
5373             $res->{spice_port} = $spice_port;
5374             if ($migrate_opts->{spice_ticket}) {
5375                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5376                     $migrate_opts->{spice_ticket});
5377                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5378             }
5379         }
5380
5381     } else {
5382         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5383             if !$statefile && $conf->{balloon};
5384
5385         foreach my $opt (keys %$conf) {
5386             next if $opt !~  m/^net\d+$/;
5387             my $nicconf = parse_net($conf->{$opt});
5388             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5389         }
5390     }
5391
5392     mon_cmd($vmid, 'qom-set',
5393                 path => "machine/peripheral/balloon0",
5394                 property => "guest-stats-polling-interval",
5395                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5396
5397     if ($resume) {
5398         print "Resumed VM, removing state\n";
5399         if (my $vmstate = $conf->{vmstate}) {
5400             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5401             PVE::Storage::vdisk_free($storecfg, $vmstate);
5402         }
5403         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5404         PVE::QemuConfig->write_config($vmid, $conf);
5405     }
5406
5407     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5408
5409     return $res;
5410 }
5411
5412 sub vm_commandline {
5413     my ($storecfg, $vmid, $snapname) = @_;
5414
5415     my $conf = PVE::QemuConfig->load_config($vmid);
5416     my $forcemachine;
5417     my $forcecpu;
5418
5419     if ($snapname) {
5420         my $snapshot = $conf->{snapshots}->{$snapname};
5421         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5422
5423         # check for machine or CPU overrides in snapshot
5424         $forcemachine = $snapshot->{runningmachine};
5425         $forcecpu = $snapshot->{runningcpu};
5426
5427         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5428
5429         $conf = $snapshot;
5430     }
5431
5432     my $defaults = load_defaults();
5433
5434     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5435         $forcemachine, $forcecpu);
5436
5437     return PVE::Tools::cmd2string($cmd);
5438 }
5439
5440 sub vm_reset {
5441     my ($vmid, $skiplock) = @_;
5442
5443     PVE::QemuConfig->lock_config($vmid, sub {
5444
5445         my $conf = PVE::QemuConfig->load_config($vmid);
5446
5447         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5448
5449         mon_cmd($vmid, "system_reset");
5450     });
5451 }
5452
5453 sub get_vm_volumes {
5454     my ($conf) = @_;
5455
5456     my $vollist = [];
5457     foreach_volid($conf, sub {
5458         my ($volid, $attr) = @_;
5459
5460         return if $volid =~ m|^/|;
5461
5462         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5463         return if !$sid;
5464
5465         push @$vollist, $volid;
5466     });
5467
5468     return $vollist;
5469 }
5470
5471 sub vm_stop_cleanup {
5472     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5473
5474     eval {
5475
5476         if (!$keepActive) {
5477             my $vollist = get_vm_volumes($conf);
5478             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5479         }
5480
5481         foreach my $ext (qw(mon qmp pid vnc qga)) {
5482             unlink "/var/run/qemu-server/${vmid}.$ext";
5483         }
5484
5485         if ($conf->{ivshmem}) {
5486             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5487             # just delete it for now, VMs which have this already open do not
5488             # are affected, but new VMs will get a separated one. If this
5489             # becomes an issue we either add some sort of ref-counting or just
5490             # add a "don't delete on stop" flag to the ivshmem format.
5491             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5492         }
5493
5494         foreach my $key (keys %$conf) {
5495             next if $key !~ m/^hostpci(\d+)$/;
5496             my $hostpciindex = $1;
5497             my $d = parse_hostpci($conf->{$key});
5498             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5499
5500             foreach my $pci (@{$d->{pciid}}) {
5501                 my $pciid = $pci->{id};
5502                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5503             }
5504         }
5505
5506         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5507     };
5508     warn $@ if $@; # avoid errors - just warn
5509 }
5510
5511 # call only in locked context
5512 sub _do_vm_stop {
5513     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5514
5515     my $pid = check_running($vmid, $nocheck);
5516     return if !$pid;
5517
5518     my $conf;
5519     if (!$nocheck) {
5520         $conf = PVE::QemuConfig->load_config($vmid);
5521         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5522         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5523             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5524             $timeout = $opts->{down} if $opts->{down};
5525         }
5526         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5527     }
5528
5529     eval {
5530         if ($shutdown) {
5531             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5532                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5533             } else {
5534                 mon_cmd($vmid, "system_powerdown");
5535             }
5536         } else {
5537             mon_cmd($vmid, "quit");
5538         }
5539     };
5540     my $err = $@;
5541
5542     if (!$err) {
5543         $timeout = 60 if !defined($timeout);
5544
5545         my $count = 0;
5546         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5547             $count++;
5548             sleep 1;
5549         }
5550
5551         if ($count >= $timeout) {
5552             if ($force) {
5553                 warn "VM still running - terminating now with SIGTERM\n";
5554                 kill 15, $pid;
5555             } else {
5556                 die "VM quit/powerdown failed - got timeout\n";
5557             }
5558         } else {
5559             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5560             return;
5561         }
5562     } else {
5563        if (!check_running($vmid, $nocheck)) {
5564             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5565             return;
5566        }
5567        if ($force) {
5568             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5569             kill 15, $pid;
5570         } else {
5571             die "VM quit/powerdown failed\n";
5572         }
5573     }
5574
5575     # wait again
5576     $timeout = 10;
5577
5578     my $count = 0;
5579     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5580         $count++;
5581         sleep 1;
5582     }
5583
5584     if ($count >= $timeout) {
5585         warn "VM still running - terminating now with SIGKILL\n";
5586         kill 9, $pid;
5587         sleep 1;
5588     }
5589
5590     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5591 }
5592
5593 # Note: use $nocheck to skip tests if VM configuration file exists.
5594 # We need that when migration VMs to other nodes (files already moved)
5595 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5596 sub vm_stop {
5597     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5598
5599     $force = 1 if !defined($force) && !$shutdown;
5600
5601     if ($migratedfrom){
5602         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5603         kill 15, $pid if $pid;
5604         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5605         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5606         return;
5607     }
5608
5609     PVE::QemuConfig->lock_config($vmid, sub {
5610         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5611    });
5612 }
5613
5614 sub vm_reboot {
5615     my ($vmid, $timeout) = @_;
5616
5617     PVE::QemuConfig->lock_config($vmid, sub {
5618         eval {
5619
5620             # only reboot if running, as qmeventd starts it again on a stop event
5621             return if !check_running($vmid);
5622
5623             create_reboot_request($vmid);
5624
5625             my $storecfg = PVE::Storage::config();
5626             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5627
5628         };
5629         if (my $err = $@) {
5630             # avoid that the next normal shutdown will be confused for a reboot
5631             clear_reboot_request($vmid);
5632             die $err;
5633         }
5634    });
5635 }
5636
5637 # note: if using the statestorage parameter, the caller has to check privileges
5638 sub vm_suspend {
5639     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5640
5641     my $conf;
5642     my $path;
5643     my $storecfg;
5644     my $vmstate;
5645
5646     PVE::QemuConfig->lock_config($vmid, sub {
5647
5648         $conf = PVE::QemuConfig->load_config($vmid);
5649
5650         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5651         PVE::QemuConfig->check_lock($conf)
5652             if !($skiplock || $is_backing_up);
5653
5654         die "cannot suspend to disk during backup\n"
5655             if $is_backing_up && $includestate;
5656
5657         if ($includestate) {
5658             $conf->{lock} = 'suspending';
5659             my $date = strftime("%Y-%m-%d", localtime(time()));
5660             $storecfg = PVE::Storage::config();
5661             if (!$statestorage) {
5662                 $statestorage = find_vmstate_storage($conf, $storecfg);
5663                 # check permissions for the storage
5664                 my $rpcenv = PVE::RPCEnvironment::get();
5665                 if ($rpcenv->{type} ne 'cli') {
5666                     my $authuser = $rpcenv->get_user();
5667                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5668                 }
5669             }
5670
5671
5672             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5673                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5674             $path = PVE::Storage::path($storecfg, $vmstate);
5675             PVE::QemuConfig->write_config($vmid, $conf);
5676         } else {
5677             mon_cmd($vmid, "stop");
5678         }
5679     });
5680
5681     if ($includestate) {
5682         # save vm state
5683         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5684
5685         eval {
5686             set_migration_caps($vmid, 1);
5687             mon_cmd($vmid, "savevm-start", statefile => $path);
5688             for(;;) {
5689                 my $state = mon_cmd($vmid, "query-savevm");
5690                 if (!$state->{status}) {
5691                     die "savevm not active\n";
5692                 } elsif ($state->{status} eq 'active') {
5693                     sleep(1);
5694                     next;
5695                 } elsif ($state->{status} eq 'completed') {
5696                     print "State saved, quitting\n";
5697                     last;
5698                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5699                     die "query-savevm failed with error '$state->{error}'\n"
5700                 } else {
5701                     die "query-savevm returned status '$state->{status}'\n";
5702                 }
5703             }
5704         };
5705         my $err = $@;
5706
5707         PVE::QemuConfig->lock_config($vmid, sub {
5708             $conf = PVE::QemuConfig->load_config($vmid);
5709             if ($err) {
5710                 # cleanup, but leave suspending lock, to indicate something went wrong
5711                 eval {
5712                     mon_cmd($vmid, "savevm-end");
5713                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5714                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5715                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5716                     PVE::QemuConfig->write_config($vmid, $conf);
5717                 };
5718                 warn $@ if $@;
5719                 die $err;
5720             }
5721
5722             die "lock changed unexpectedly\n"
5723                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5724
5725             mon_cmd($vmid, "quit");
5726             $conf->{lock} = 'suspended';
5727             PVE::QemuConfig->write_config($vmid, $conf);
5728         });
5729     }
5730 }
5731
5732 sub vm_resume {
5733     my ($vmid, $skiplock, $nocheck) = @_;
5734
5735     PVE::QemuConfig->lock_config($vmid, sub {
5736         my $res = mon_cmd($vmid, 'query-status');
5737         my $resume_cmd = 'cont';
5738         my $reset = 0;
5739
5740         if ($res->{status}) {
5741             return if $res->{status} eq 'running'; # job done, go home
5742             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5743             $reset = 1 if $res->{status} eq 'shutdown';
5744         }
5745
5746         if (!$nocheck) {
5747
5748             my $conf = PVE::QemuConfig->load_config($vmid);
5749
5750             PVE::QemuConfig->check_lock($conf)
5751                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5752         }
5753
5754         if ($reset) {
5755             # required if a VM shuts down during a backup and we get a resume
5756             # request before the backup finishes for example
5757             mon_cmd($vmid, "system_reset");
5758         }
5759         mon_cmd($vmid, $resume_cmd);
5760     });
5761 }
5762
5763 sub vm_sendkey {
5764     my ($vmid, $skiplock, $key) = @_;
5765
5766     PVE::QemuConfig->lock_config($vmid, sub {
5767
5768         my $conf = PVE::QemuConfig->load_config($vmid);
5769
5770         # there is no qmp command, so we use the human monitor command
5771         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5772         die $res if $res ne '';
5773     });
5774 }
5775
5776 # vzdump restore implementaion
5777
5778 sub tar_archive_read_firstfile {
5779     my $archive = shift;
5780
5781     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5782
5783     # try to detect archive type first
5784     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5785         die "unable to open file '$archive'\n";
5786     my $firstfile = <$fh>;
5787     kill 15, $pid;
5788     close $fh;
5789
5790     die "ERROR: archive contaions no data\n" if !$firstfile;
5791     chomp $firstfile;
5792
5793     return $firstfile;
5794 }
5795
5796 sub tar_restore_cleanup {
5797     my ($storecfg, $statfile) = @_;
5798
5799     print STDERR "starting cleanup\n";
5800
5801     if (my $fd = IO::File->new($statfile, "r")) {
5802         while (defined(my $line = <$fd>)) {
5803             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5804                 my $volid = $2;
5805                 eval {
5806                     if ($volid =~ m|^/|) {
5807                         unlink $volid || die 'unlink failed\n';
5808                     } else {
5809                         PVE::Storage::vdisk_free($storecfg, $volid);
5810                     }
5811                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5812                 };
5813                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5814             } else {
5815                 print STDERR "unable to parse line in statfile - $line";
5816             }
5817         }
5818         $fd->close();
5819     }
5820 }
5821
5822 sub restore_file_archive {
5823     my ($archive, $vmid, $user, $opts) = @_;
5824
5825     return restore_vma_archive($archive, $vmid, $user, $opts)
5826         if $archive eq '-';
5827
5828     my $info = PVE::Storage::archive_info($archive);
5829     my $format = $opts->{format} // $info->{format};
5830     my $comp = $info->{compression};
5831
5832     # try to detect archive format
5833     if ($format eq 'tar') {
5834         return restore_tar_archive($archive, $vmid, $user, $opts);
5835     } else {
5836         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5837     }
5838 }
5839
5840 # hepler to remove disks that will not be used after restore
5841 my $restore_cleanup_oldconf = sub {
5842     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5843
5844     PVE::QemuConfig->foreach_volume($oldconf, sub {
5845         my ($ds, $drive) = @_;
5846
5847         return if drive_is_cdrom($drive, 1);
5848
5849         my $volid = $drive->{file};
5850         return if !$volid || $volid =~ m|^/|;
5851
5852         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5853         return if !$path || !$owner || ($owner != $vmid);
5854
5855         # Note: only delete disk we want to restore
5856         # other volumes will become unused
5857         if ($virtdev_hash->{$ds}) {
5858             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5859             if (my $err = $@) {
5860                 warn $err;
5861             }
5862         }
5863     });
5864
5865     # delete vmstate files, after the restore we have no snapshots anymore
5866     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5867         my $snap = $oldconf->{snapshots}->{$snapname};
5868         if ($snap->{vmstate}) {
5869             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5870             if (my $err = $@) {
5871                 warn $err;
5872             }
5873         }
5874     }
5875 };
5876
5877 # Helper to parse vzdump backup device hints
5878 #
5879 # $rpcenv: Environment, used to ckeck storage permissions
5880 # $user: User ID, to check storage permissions
5881 # $storecfg: Storage configuration
5882 # $fh: the file handle for reading the configuration
5883 # $devinfo: should contain device sizes for all backu-up'ed devices
5884 # $options: backup options (pool, default storage)
5885 #
5886 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5887 my $parse_backup_hints = sub {
5888     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5889
5890     my $virtdev_hash = {};
5891
5892     while (defined(my $line = <$fh>)) {
5893         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5894             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5895             die "archive does not contain data for drive '$virtdev'\n"
5896                 if !$devinfo->{$devname};
5897
5898             if (defined($options->{storage})) {
5899                 $storeid = $options->{storage} || 'local';
5900             } elsif (!$storeid) {
5901                 $storeid = 'local';
5902             }
5903             $format = 'raw' if !$format;
5904             $devinfo->{$devname}->{devname} = $devname;
5905             $devinfo->{$devname}->{virtdev} = $virtdev;
5906             $devinfo->{$devname}->{format} = $format;
5907             $devinfo->{$devname}->{storeid} = $storeid;
5908
5909             # check permission on storage
5910             my $pool = $options->{pool}; # todo: do we need that?
5911             if ($user ne 'root@pam') {
5912                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5913             }
5914
5915             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5916         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5917             my $virtdev = $1;
5918             my $drive = parse_drive($virtdev, $2);
5919             if (drive_is_cloudinit($drive)) {
5920                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5921                 $storeid = $options->{storage} if defined ($options->{storage});
5922                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5923                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5924
5925                 $virtdev_hash->{$virtdev} = {
5926                     format => $format,
5927                     storeid => $storeid,
5928                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5929                     is_cloudinit => 1,
5930                 };
5931             }
5932         }
5933     }
5934
5935     return $virtdev_hash;
5936 };
5937
5938 # Helper to allocate and activate all volumes required for a restore
5939 #
5940 # $storecfg: Storage configuration
5941 # $virtdev_hash: as returned by parse_backup_hints()
5942 #
5943 # Returns: { $virtdev => $volid }
5944 my $restore_allocate_devices = sub {
5945     my ($storecfg, $virtdev_hash, $vmid) = @_;
5946
5947     my $map = {};
5948     foreach my $virtdev (sort keys %$virtdev_hash) {
5949         my $d = $virtdev_hash->{$virtdev};
5950         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5951         my $storeid = $d->{storeid};
5952         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5953
5954         # test if requested format is supported
5955         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5956         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5957         $d->{format} = $defFormat if !$supported;
5958
5959         my $name;
5960         if ($d->{is_cloudinit}) {
5961             $name = "vm-$vmid-cloudinit";
5962             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5963             if ($scfg->{path}) {
5964                 $name .= ".$d->{format}";
5965             }
5966         }
5967
5968         my $volid = PVE::Storage::vdisk_alloc(
5969             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5970
5971         print STDERR "new volume ID is '$volid'\n";
5972         $d->{volid} = $volid;
5973
5974         PVE::Storage::activate_volumes($storecfg, [$volid]);
5975
5976         $map->{$virtdev} = $volid;
5977     }
5978
5979     return $map;
5980 };
5981
5982 sub restore_update_config_line {
5983     my ($cookie, $map, $line, $unique) = @_;
5984
5985     return '' if $line =~ m/^\#qmdump\#/;
5986     return '' if $line =~ m/^\#vzdump\#/;
5987     return '' if $line =~ m/^lock:/;
5988     return '' if $line =~ m/^unused\d+:/;
5989     return '' if $line =~ m/^parent:/;
5990
5991     my $res = '';
5992
5993     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5994     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5995         # try to convert old 1.X settings
5996         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5997         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5998             my ($model, $macaddr) = split(/\=/, $devconfig);
5999             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6000             my $net = {
6001                 model => $model,
6002                 bridge => "vmbr$ind",
6003                 macaddr => $macaddr,
6004             };
6005             my $netstr = print_net($net);
6006
6007             $res .= "net$cookie->{netcount}: $netstr\n";
6008             $cookie->{netcount}++;
6009         }
6010     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6011         my ($id, $netstr) = ($1, $2);
6012         my $net = parse_net($netstr);
6013         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6014         $netstr = print_net($net);
6015         $res .= "$id: $netstr\n";
6016     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6017         my $virtdev = $1;
6018         my $value = $3;
6019         my $di = parse_drive($virtdev, $value);
6020         if (defined($di->{backup}) && !$di->{backup}) {
6021             $res .= "#$line";
6022         } elsif ($map->{$virtdev}) {
6023             delete $di->{format}; # format can change on restore
6024             $di->{file} = $map->{$virtdev};
6025             $value = print_drive($di);
6026             $res .= "$virtdev: $value\n";
6027         } else {
6028             $res .= $line;
6029         }
6030     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6031         my $vmgenid = $1;
6032         if ($vmgenid ne '0') {
6033             # always generate a new vmgenid if there was a valid one setup
6034             $vmgenid = generate_uuid();
6035         }
6036         $res .= "vmgenid: $vmgenid\n";
6037     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6038         my ($uuid, $uuid_str);
6039         UUID::generate($uuid);
6040         UUID::unparse($uuid, $uuid_str);
6041         my $smbios1 = parse_smbios1($2);
6042         $smbios1->{uuid} = $uuid_str;
6043         $res .= $1.print_smbios1($smbios1)."\n";
6044     } else {
6045         $res .= $line;
6046     }
6047
6048     return $res;
6049 }
6050
6051 my $restore_deactivate_volumes = sub {
6052     my ($storecfg, $devinfo) = @_;
6053
6054     my $vollist = [];
6055     foreach my $devname (keys %$devinfo) {
6056         my $volid = $devinfo->{$devname}->{volid};
6057         push @$vollist, $volid if $volid;
6058     }
6059
6060     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6061 };
6062
6063 my $restore_destroy_volumes = sub {
6064     my ($storecfg, $devinfo) = @_;
6065
6066     foreach my $devname (keys %$devinfo) {
6067         my $volid = $devinfo->{$devname}->{volid};
6068         next if !$volid;
6069         eval {
6070             if ($volid =~ m|^/|) {
6071                 unlink $volid || die 'unlink failed\n';
6072             } else {
6073                 PVE::Storage::vdisk_free($storecfg, $volid);
6074             }
6075             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6076         };
6077         print STDERR "unable to cleanup '$volid' - $@" if $@;
6078     }
6079 };
6080
6081 # FIXME For PVE 7.0, remove $content_type and always use 'images'
6082 sub scan_volids {
6083     my ($cfg, $vmid, $content_type) = @_;
6084
6085     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, $content_type);
6086
6087     my $volid_hash = {};
6088     foreach my $storeid (keys %$info) {
6089         foreach my $item (@{$info->{$storeid}}) {
6090             next if !($item->{volid} && $item->{size});
6091             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6092             $volid_hash->{$item->{volid}} = $item;
6093         }
6094     }
6095
6096     return $volid_hash;
6097 }
6098
6099 sub update_disk_config {
6100     my ($vmid, $conf, $volid_hash) = @_;
6101
6102     my $changes;
6103     my $prefix = "VM $vmid";
6104
6105     # used and unused disks
6106     my $referenced = {};
6107
6108     # Note: it is allowed to define multiple storages with same path (alias), so
6109     # we need to check both 'volid' and real 'path' (two different volid can point
6110     # to the same path).
6111
6112     my $referencedpath = {};
6113
6114     # update size info
6115     PVE::QemuConfig->foreach_volume($conf, sub {
6116         my ($opt, $drive) = @_;
6117
6118         my $volid = $drive->{file};
6119         return if !$volid;
6120         my $volume = $volid_hash->{$volid};
6121
6122         # mark volid as "in-use" for next step
6123         $referenced->{$volid} = 1;
6124         if ($volume && (my $path = $volume->{path})) {
6125             $referencedpath->{$path} = 1;
6126         }
6127
6128         return if drive_is_cdrom($drive);
6129         return if !$volume;
6130
6131         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6132         if (defined($updated)) {
6133             $changes = 1;
6134             $conf->{$opt} = print_drive($updated);
6135             print "$prefix ($opt): $msg\n";
6136         }
6137     });
6138
6139     # remove 'unusedX' entry if volume is used
6140     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6141         my ($opt, $drive) = @_;
6142
6143         my $volid = $drive->{file};
6144         return if !$volid;
6145
6146         my $path;
6147         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6148         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6149             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6150             $changes = 1;
6151             delete $conf->{$opt};
6152         }
6153
6154         $referenced->{$volid} = 1;
6155         $referencedpath->{$path} = 1 if $path;
6156     });
6157
6158     foreach my $volid (sort keys %$volid_hash) {
6159         next if $volid =~ m/vm-$vmid-state-/;
6160         next if $referenced->{$volid};
6161         my $path = $volid_hash->{$volid}->{path};
6162         next if !$path; # just to be sure
6163         next if $referencedpath->{$path};
6164         $changes = 1;
6165         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6166         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6167         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6168     }
6169
6170     return $changes;
6171 }
6172
6173 sub rescan {
6174     my ($vmid, $nolock, $dryrun) = @_;
6175
6176     my $cfg = PVE::Storage::config();
6177
6178     print "rescan volumes...\n";
6179     my $volid_hash = scan_volids($cfg, $vmid, 'images');
6180
6181     my $updatefn =  sub {
6182         my ($vmid) = @_;
6183
6184         my $conf = PVE::QemuConfig->load_config($vmid);
6185
6186         PVE::QemuConfig->check_lock($conf);
6187
6188         my $vm_volids = {};
6189         foreach my $volid (keys %$volid_hash) {
6190             my $info = $volid_hash->{$volid};
6191             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6192         }
6193
6194         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6195
6196         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6197     };
6198
6199     if (defined($vmid)) {
6200         if ($nolock) {
6201             &$updatefn($vmid);
6202         } else {
6203             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6204         }
6205     } else {
6206         my $vmlist = config_list();
6207         foreach my $vmid (keys %$vmlist) {
6208             if ($nolock) {
6209                 &$updatefn($vmid);
6210             } else {
6211                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6212             }
6213         }
6214     }
6215 }
6216
6217 sub restore_proxmox_backup_archive {
6218     my ($archive, $vmid, $user, $options) = @_;
6219
6220     my $storecfg = PVE::Storage::config();
6221
6222     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6223     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6224
6225     my $fingerprint = $scfg->{fingerprint};
6226     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6227
6228     my $repo = PVE::PBSClient::get_repository($scfg);
6229
6230     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6231     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6232     local $ENV{PBS_PASSWORD} = $password;
6233     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6234
6235     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6236         PVE::Storage::parse_volname($storecfg, $archive);
6237
6238     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6239
6240     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6241
6242     my $tmpdir = "/var/tmp/vzdumptmp$$";
6243     rmtree $tmpdir;
6244     mkpath $tmpdir;
6245
6246     my $conffile = PVE::QemuConfig->config_file($vmid);
6247      # disable interrupts (always do cleanups)
6248     local $SIG{INT} =
6249         local $SIG{TERM} =
6250         local $SIG{QUIT} =
6251         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6252
6253     # Note: $oldconf is undef if VM does not exists
6254     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6255     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6256     my $new_conf_raw = '';
6257
6258     my $rpcenv = PVE::RPCEnvironment::get();
6259     my $devinfo = {};
6260
6261     eval {
6262         # enable interrupts
6263         local $SIG{INT} =
6264             local $SIG{TERM} =
6265             local $SIG{QUIT} =
6266             local $SIG{HUP} =
6267             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6268
6269         my $cfgfn = "$tmpdir/qemu-server.conf";
6270         my $firewall_config_fn = "$tmpdir/fw.conf";
6271         my $index_fn = "$tmpdir/index.json";
6272
6273         my $cmd = "restore";
6274
6275         my $param = [$pbs_backup_name, "index.json", $index_fn];
6276         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6277         my $index = PVE::Tools::file_get_contents($index_fn);
6278         $index = decode_json($index);
6279
6280         # print Dumper($index);
6281         foreach my $info (@{$index->{files}}) {
6282             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6283                 my $devname = $1;
6284                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6285                     $devinfo->{$devname}->{size} = $1;
6286                 } else {
6287                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6288                 }
6289             }
6290         }
6291
6292         my $is_qemu_server_backup = scalar(
6293             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6294         );
6295         if (!$is_qemu_server_backup) {
6296             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6297         }
6298         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6299
6300         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6301         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6302
6303         if ($has_firewall_config) {
6304             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6305             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6306
6307             my $pve_firewall_dir = '/etc/pve/firewall';
6308             mkdir $pve_firewall_dir; # make sure the dir exists
6309             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6310         }
6311
6312         my $fh = IO::File->new($cfgfn, "r") ||
6313             die "unable to read qemu-server.conf - $!\n";
6314
6315         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6316
6317         # fixme: rate limit?
6318
6319         # create empty/temp config
6320         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6321
6322         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6323
6324         # allocate volumes
6325         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6326
6327         if (!$options->{live}) {
6328             foreach my $virtdev (sort keys %$virtdev_hash) {
6329                 my $d = $virtdev_hash->{$virtdev};
6330                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6331
6332                 my $volid = $d->{volid};
6333
6334                 my $path = PVE::Storage::path($storecfg, $volid);
6335
6336                 my $pbs_restore_cmd = [
6337                     '/usr/bin/pbs-restore',
6338                     '--repository', $repo,
6339                     $pbs_backup_name,
6340                     "$d->{devname}.img.fidx",
6341                     $path,
6342                     '--verbose',
6343                     ];
6344
6345                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6346                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6347
6348                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6349                     push @$pbs_restore_cmd, '--skip-zero';
6350                 }
6351
6352                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6353                 print "restore proxmox backup image: $dbg_cmdstring\n";
6354                 run_command($pbs_restore_cmd);
6355             }
6356         }
6357
6358         $fh->seek(0, 0) || die "seek failed - $!\n";
6359
6360         my $cookie = { netcount => 0 };
6361         while (defined(my $line = <$fh>)) {
6362             $new_conf_raw .= restore_update_config_line(
6363                 $cookie,
6364                 $map,
6365                 $line,
6366                 $options->{unique},
6367             );
6368         }
6369
6370         $fh->close();
6371     };
6372     my $err = $@;
6373
6374     if ($err || !$options->{live}) {
6375         $restore_deactivate_volumes->($storecfg, $devinfo);
6376     }
6377
6378     rmtree $tmpdir;
6379
6380     if ($err) {
6381         $restore_destroy_volumes->($storecfg, $devinfo);
6382         die $err;
6383     }
6384
6385     if ($options->{live}) {
6386         # keep lock during live-restore
6387         $new_conf_raw .= "\nlock: create";
6388     }
6389
6390     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6391
6392     PVE::Cluster::cfs_update(); # make sure we read new file
6393
6394     eval { rescan($vmid, 1); };
6395     warn $@ if $@;
6396
6397     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6398
6399     if ($options->{live}) {
6400         # enable interrupts
6401         local $SIG{INT} =
6402             local $SIG{TERM} =
6403             local $SIG{QUIT} =
6404             local $SIG{HUP} =
6405             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6406
6407         my $conf = PVE::QemuConfig->load_config($vmid);
6408         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6409
6410         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6411
6412         PVE::QemuConfig->remove_lock($vmid, "create");
6413     }
6414 }
6415
6416 sub pbs_live_restore {
6417     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6418
6419     print "Starting VM for live-restore\n";
6420
6421     my $pbs_backing = {};
6422     for my $ds (keys %$restored_disks) {
6423         $ds =~ m/^drive-(.*)$/;
6424         $pbs_backing->{$1} = {
6425             repository => $repo,
6426             snapshot => $snap,
6427             archive => "$ds.img.fidx",
6428         };
6429         $pbs_backing->{$1}->{keyfile} = $keyfile if -e $keyfile;
6430     }
6431
6432     my $drives_streamed = 0;
6433     eval {
6434         # make sure HA doesn't interrupt our restore by stopping the VM
6435         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6436             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6437         }
6438
6439         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6440         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6441         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6442
6443         my $qmeventd_fd = register_qmeventd_handle($vmid);
6444
6445         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6446         # this will effectively collapse the backing image chain consisting of
6447         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6448         # removes itself once all backing images vanish with 'auto-remove=on')
6449         my $jobs = {};
6450         for my $ds (sort keys %$restored_disks) {
6451             my $job_id = "restore-$ds";
6452             mon_cmd($vmid, 'block-stream',
6453                 'job-id' => $job_id,
6454                 device => "$ds",
6455             );
6456             $jobs->{$job_id} = {};
6457         }
6458
6459         mon_cmd($vmid, 'cont');
6460         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6461
6462         print "restore-drive jobs finished successfully, removing all tracking block devices"
6463             ." to disconnect from Proxmox Backup Server\n";
6464
6465         for my $ds (sort keys %$restored_disks) {
6466             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6467         }
6468
6469         close($qmeventd_fd);
6470     };
6471
6472     my $err = $@;
6473
6474     if ($err) {
6475         warn "An error occured during live-restore: $err\n";
6476         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6477         die "live-restore failed\n";
6478     }
6479 }
6480
6481 sub restore_vma_archive {
6482     my ($archive, $vmid, $user, $opts, $comp) = @_;
6483
6484     my $readfrom = $archive;
6485
6486     my $cfg = PVE::Storage::config();
6487     my $commands = [];
6488     my $bwlimit = $opts->{bwlimit};
6489
6490     my $dbg_cmdstring = '';
6491     my $add_pipe = sub {
6492         my ($cmd) = @_;
6493         push @$commands, $cmd;
6494         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6495         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6496         $readfrom = '-';
6497     };
6498
6499     my $input = undef;
6500     if ($archive eq '-') {
6501         $input = '<&STDIN';
6502     } else {
6503         # If we use a backup from a PVE defined storage we also consider that
6504         # storage's rate limit:
6505         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6506         if (defined($volid)) {
6507             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6508             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6509             if ($readlimit) {
6510                 print STDERR "applying read rate limit: $readlimit\n";
6511                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6512                 $add_pipe->($cstream);
6513             }
6514         }
6515     }
6516
6517     if ($comp) {
6518         my $info = PVE::Storage::decompressor_info('vma', $comp);
6519         my $cmd = $info->{decompressor};
6520         push @$cmd, $readfrom;
6521         $add_pipe->($cmd);
6522     }
6523
6524     my $tmpdir = "/var/tmp/vzdumptmp$$";
6525     rmtree $tmpdir;
6526
6527     # disable interrupts (always do cleanups)
6528     local $SIG{INT} =
6529         local $SIG{TERM} =
6530         local $SIG{QUIT} =
6531         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6532
6533     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6534     POSIX::mkfifo($mapfifo, 0600);
6535     my $fifofh;
6536     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6537
6538     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6539
6540     my $oldtimeout;
6541     my $timeout = 5;
6542
6543     my $devinfo = {};
6544
6545     my $rpcenv = PVE::RPCEnvironment::get();
6546
6547     my $conffile = PVE::QemuConfig->config_file($vmid);
6548
6549     # Note: $oldconf is undef if VM does not exist
6550     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6551     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6552     my $new_conf_raw = '';
6553
6554     my %storage_limits;
6555
6556     my $print_devmap = sub {
6557         my $cfgfn = "$tmpdir/qemu-server.conf";
6558
6559         # we can read the config - that is already extracted
6560         my $fh = IO::File->new($cfgfn, "r") ||
6561             die "unable to read qemu-server.conf - $!\n";
6562
6563         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6564         if (-f $fwcfgfn) {
6565             my $pve_firewall_dir = '/etc/pve/firewall';
6566             mkdir $pve_firewall_dir; # make sure the dir exists
6567             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6568         }
6569
6570         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6571
6572         foreach my $info (values %{$virtdev_hash}) {
6573             my $storeid = $info->{storeid};
6574             next if defined($storage_limits{$storeid});
6575
6576             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6577             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6578             $storage_limits{$storeid} = $limit * 1024;
6579         }
6580
6581         foreach my $devname (keys %$devinfo) {
6582             die "found no device mapping information for device '$devname'\n"
6583                 if !$devinfo->{$devname}->{virtdev};
6584         }
6585
6586         # create empty/temp config
6587         if ($oldconf) {
6588             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6589             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6590         }
6591
6592         # allocate volumes
6593         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6594
6595         # print restore information to $fifofh
6596         foreach my $virtdev (sort keys %$virtdev_hash) {
6597             my $d = $virtdev_hash->{$virtdev};
6598             next if $d->{is_cloudinit}; # no need to restore cloudinit
6599
6600             my $storeid = $d->{storeid};
6601             my $volid = $d->{volid};
6602
6603             my $map_opts = '';
6604             if (my $limit = $storage_limits{$storeid}) {
6605                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6606             }
6607
6608             my $write_zeros = 1;
6609             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6610                 $write_zeros = 0;
6611             }
6612
6613             my $path = PVE::Storage::path($cfg, $volid);
6614
6615             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6616
6617             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6618         }
6619
6620         $fh->seek(0, 0) || die "seek failed - $!\n";
6621
6622         my $cookie = { netcount => 0 };
6623         while (defined(my $line = <$fh>)) {
6624             $new_conf_raw .= restore_update_config_line(
6625                 $cookie,
6626                 $map,
6627                 $line,
6628                 $opts->{unique},
6629             );
6630         }
6631
6632         $fh->close();
6633     };
6634
6635     eval {
6636         # enable interrupts
6637         local $SIG{INT} =
6638             local $SIG{TERM} =
6639             local $SIG{QUIT} =
6640             local $SIG{HUP} =
6641             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6642         local $SIG{ALRM} = sub { die "got timeout\n"; };
6643
6644         $oldtimeout = alarm($timeout);
6645
6646         my $parser = sub {
6647             my $line = shift;
6648
6649             print "$line\n";
6650
6651             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6652                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6653                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6654             } elsif ($line =~ m/^CTIME: /) {
6655                 # we correctly received the vma config, so we can disable
6656                 # the timeout now for disk allocation (set to 10 minutes, so
6657                 # that we always timeout if something goes wrong)
6658                 alarm(600);
6659                 &$print_devmap();
6660                 print $fifofh "done\n";
6661                 my $tmp = $oldtimeout || 0;
6662                 $oldtimeout = undef;
6663                 alarm($tmp);
6664                 close($fifofh);
6665                 $fifofh = undef;
6666             }
6667         };
6668
6669         print "restore vma archive: $dbg_cmdstring\n";
6670         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6671     };
6672     my $err = $@;
6673
6674     alarm($oldtimeout) if $oldtimeout;
6675
6676     $restore_deactivate_volumes->($cfg, $devinfo);
6677
6678     close($fifofh) if $fifofh;
6679     unlink $mapfifo;
6680     rmtree $tmpdir;
6681
6682     if ($err) {
6683         $restore_destroy_volumes->($cfg, $devinfo);
6684         die $err;
6685     }
6686
6687     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6688
6689     PVE::Cluster::cfs_update(); # make sure we read new file
6690
6691     eval { rescan($vmid, 1); };
6692     warn $@ if $@;
6693
6694     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6695 }
6696
6697 sub restore_tar_archive {
6698     my ($archive, $vmid, $user, $opts) = @_;
6699
6700     if ($archive ne '-') {
6701         my $firstfile = tar_archive_read_firstfile($archive);
6702         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6703             if $firstfile ne 'qemu-server.conf';
6704     }
6705
6706     my $storecfg = PVE::Storage::config();
6707
6708     # avoid zombie disks when restoring over an existing VM -> cleanup first
6709     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6710     # skiplock=1 because qmrestore has set the 'create' lock itself already
6711     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6712     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6713
6714     my $tocmd = "/usr/lib/qemu-server/qmextract";
6715
6716     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6717     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6718     $tocmd .= ' --prealloc' if $opts->{prealloc};
6719     $tocmd .= ' --info' if $opts->{info};
6720
6721     # tar option "xf" does not autodetect compression when read from STDIN,
6722     # so we pipe to zcat
6723     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6724         PVE::Tools::shellquote("--to-command=$tocmd");
6725
6726     my $tmpdir = "/var/tmp/vzdumptmp$$";
6727     mkpath $tmpdir;
6728
6729     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6730     local $ENV{VZDUMP_VMID} = $vmid;
6731     local $ENV{VZDUMP_USER} = $user;
6732
6733     my $conffile = PVE::QemuConfig->config_file($vmid);
6734     my $new_conf_raw = '';
6735
6736     # disable interrupts (always do cleanups)
6737     local $SIG{INT} =
6738         local $SIG{TERM} =
6739         local $SIG{QUIT} =
6740         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6741
6742     eval {
6743         # enable interrupts
6744         local $SIG{INT} =
6745             local $SIG{TERM} =
6746             local $SIG{QUIT} =
6747             local $SIG{HUP} =
6748             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6749
6750         if ($archive eq '-') {
6751             print "extracting archive from STDIN\n";
6752             run_command($cmd, input => "<&STDIN");
6753         } else {
6754             print "extracting archive '$archive'\n";
6755             run_command($cmd);
6756         }
6757
6758         return if $opts->{info};
6759
6760         # read new mapping
6761         my $map = {};
6762         my $statfile = "$tmpdir/qmrestore.stat";
6763         if (my $fd = IO::File->new($statfile, "r")) {
6764             while (defined (my $line = <$fd>)) {
6765                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6766                     $map->{$1} = $2 if $1;
6767                 } else {
6768                     print STDERR "unable to parse line in statfile - $line\n";
6769                 }
6770             }
6771             $fd->close();
6772         }
6773
6774         my $confsrc = "$tmpdir/qemu-server.conf";
6775
6776         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6777
6778         my $cookie = { netcount => 0 };
6779         while (defined (my $line = <$srcfd>)) {
6780             $new_conf_raw .= restore_update_config_line(
6781                 $cookie,
6782                 $map,
6783                 $line,
6784                 $opts->{unique},
6785             );
6786         }
6787
6788         $srcfd->close();
6789     };
6790     if (my $err = $@) {
6791         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6792         die $err;
6793     }
6794
6795     rmtree $tmpdir;
6796
6797     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6798
6799     PVE::Cluster::cfs_update(); # make sure we read new file
6800
6801     eval { rescan($vmid, 1); };
6802     warn $@ if $@;
6803 };
6804
6805 sub foreach_storage_used_by_vm {
6806     my ($conf, $func) = @_;
6807
6808     my $sidhash = {};
6809
6810     PVE::QemuConfig->foreach_volume($conf, sub {
6811         my ($ds, $drive) = @_;
6812         return if drive_is_cdrom($drive);
6813
6814         my $volid = $drive->{file};
6815
6816         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6817         $sidhash->{$sid} = $sid if $sid;
6818     });
6819
6820     foreach my $sid (sort keys %$sidhash) {
6821         &$func($sid);
6822     }
6823 }
6824
6825 my $qemu_snap_storage = {
6826     rbd => 1,
6827 };
6828 sub do_snapshots_with_qemu {
6829     my ($storecfg, $volid) = @_;
6830
6831     my $storage_name = PVE::Storage::parse_volume_id($volid);
6832     my $scfg = $storecfg->{ids}->{$storage_name};
6833     die "could not find storage '$storage_name'\n" if !defined($scfg);
6834
6835     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6836         return 1;
6837     }
6838
6839     if ($volid =~ m/\.(qcow2|qed)$/){
6840         return 1;
6841     }
6842
6843     return;
6844 }
6845
6846 sub qga_check_running {
6847     my ($vmid, $nowarn) = @_;
6848
6849     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6850     if ($@) {
6851         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6852         return 0;
6853     }
6854     return 1;
6855 }
6856
6857 sub template_create {
6858     my ($vmid, $conf, $disk) = @_;
6859
6860     my $storecfg = PVE::Storage::config();
6861
6862     PVE::QemuConfig->foreach_volume($conf, sub {
6863         my ($ds, $drive) = @_;
6864
6865         return if drive_is_cdrom($drive);
6866         return if $disk && $ds ne $disk;
6867
6868         my $volid = $drive->{file};
6869         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6870
6871         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6872         $drive->{file} = $voliddst;
6873         $conf->{$ds} = print_drive($drive);
6874         PVE::QemuConfig->write_config($vmid, $conf);
6875     });
6876 }
6877
6878 sub convert_iscsi_path {
6879     my ($path) = @_;
6880
6881     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6882         my $portal = $1;
6883         my $target = $2;
6884         my $lun = $3;
6885
6886         my $initiator_name = get_initiator_name();
6887
6888         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6889                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6890     }
6891
6892     die "cannot convert iscsi path '$path', unkown format\n";
6893 }
6894
6895 sub qemu_img_convert {
6896     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6897
6898     my $storecfg = PVE::Storage::config();
6899     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6900     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6901
6902     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6903
6904     my $cachemode;
6905     my $src_path;
6906     my $src_is_iscsi = 0;
6907     my $src_format;
6908
6909     if ($src_storeid) {
6910         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6911         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6912         $src_format = qemu_img_format($src_scfg, $src_volname);
6913         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6914         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6915         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6916     } elsif (-f $src_volid) {
6917         $src_path = $src_volid;
6918         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6919             $src_format = $1;
6920         }
6921     }
6922
6923     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6924
6925     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6926     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6927     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6928     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6929
6930     my $cmd = [];
6931     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6932     push @$cmd, '-l', "snapshot.name=$snapname"
6933         if $snapname && $src_format && $src_format eq "qcow2";
6934     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6935     push @$cmd, '-T', $cachemode if defined($cachemode);
6936
6937     if ($src_is_iscsi) {
6938         push @$cmd, '--image-opts';
6939         $src_path = convert_iscsi_path($src_path);
6940     } elsif ($src_format) {
6941         push @$cmd, '-f', $src_format;
6942     }
6943
6944     if ($dst_is_iscsi) {
6945         push @$cmd, '--target-image-opts';
6946         $dst_path = convert_iscsi_path($dst_path);
6947     } else {
6948         push @$cmd, '-O', $dst_format;
6949     }
6950
6951     push @$cmd, $src_path;
6952
6953     if (!$dst_is_iscsi && $is_zero_initialized) {
6954         push @$cmd, "zeroinit:$dst_path";
6955     } else {
6956         push @$cmd, $dst_path;
6957     }
6958
6959     my $parser = sub {
6960         my $line = shift;
6961         if($line =~ m/\((\S+)\/100\%\)/){
6962             my $percent = $1;
6963             my $transferred = int($size * $percent / 100);
6964             my $total_h = render_bytes($size, 1);
6965             my $transferred_h = render_bytes($transferred, 1);
6966
6967             print "transferred $transferred_h of $total_h ($percent%)\n";
6968         }
6969
6970     };
6971
6972     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6973     my $err = $@;
6974     die "copy failed: $err" if $err;
6975 }
6976
6977 sub qemu_img_format {
6978     my ($scfg, $volname) = @_;
6979
6980     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6981         return $1;
6982     } else {
6983         return "raw";
6984     }
6985 }
6986
6987 sub qemu_drive_mirror {
6988     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6989
6990     $jobs = {} if !$jobs;
6991
6992     my $qemu_target;
6993     my $format;
6994     $jobs->{"drive-$drive"} = {};
6995
6996     if ($dst_volid =~ /^nbd:/) {
6997         $qemu_target = $dst_volid;
6998         $format = "nbd";
6999     } else {
7000         my $storecfg = PVE::Storage::config();
7001         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7002
7003         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7004
7005         $format = qemu_img_format($dst_scfg, $dst_volname);
7006
7007         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7008
7009         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7010     }
7011
7012     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7013     $opts->{format} = $format if $format;
7014
7015     if (defined($src_bitmap)) {
7016         $opts->{sync} = 'incremental';
7017         $opts->{bitmap} = $src_bitmap;
7018         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7019     }
7020
7021     if (defined($bwlimit)) {
7022         $opts->{speed} = $bwlimit * 1024;
7023         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7024     } else {
7025         print "drive mirror is starting for drive-$drive\n";
7026     }
7027
7028     # if a job already runs for this device we get an error, catch it for cleanup
7029     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7030     if (my $err = $@) {
7031         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7032         warn "$@\n" if $@;
7033         die "mirroring error: $err\n";
7034     }
7035
7036     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7037 }
7038
7039 # $completion can be either
7040 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7041 # 'cancel': wait until all jobs are ready, block-job-cancel them
7042 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7043 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7044 sub qemu_drive_mirror_monitor {
7045     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7046
7047     $completion //= 'complete';
7048     $op //= "mirror";
7049
7050     eval {
7051         my $err_complete = 0;
7052
7053         my $starttime = time ();
7054         while (1) {
7055             die "block job ('$op') timed out\n" if $err_complete > 300;
7056
7057             my $stats = mon_cmd($vmid, "query-block-jobs");
7058             my $ctime = time();
7059
7060             my $running_jobs = {};
7061             for my $stat (@$stats) {
7062                 next if $stat->{type} ne $op;
7063                 $running_jobs->{$stat->{device}} = $stat;
7064             }
7065
7066             my $readycounter = 0;
7067
7068             for my $job_id (sort keys %$jobs) {
7069                 my $job = $running_jobs->{$job_id};
7070
7071                 my $vanished = !defined($job);
7072                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7073                 if($complete || ($vanished && $completion eq 'auto')) {
7074                     print "$job_id: $op-job finished\n";
7075                     delete $jobs->{$job_id};
7076                     next;
7077                 }
7078
7079                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7080
7081                 my $busy = $job->{busy};
7082                 my $ready = $job->{ready};
7083                 if (my $total = $job->{len}) {
7084                     my $transferred = $job->{offset} || 0;
7085                     my $remaining = $total - $transferred;
7086                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7087
7088                     my $duration = $ctime - $starttime;
7089                     my $total_h = render_bytes($total, 1);
7090                     my $transferred_h = render_bytes($transferred, 1);
7091
7092                     my $status = sprintf(
7093                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7094                         render_duration($duration),
7095                     );
7096
7097                     if ($ready) {
7098                         if ($busy) {
7099                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7100                         } else {
7101                             $status .= ", ready";
7102                         }
7103                     }
7104                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7105                     $jobs->{$job_id}->{ready} = $ready;
7106                 }
7107
7108                 $readycounter++ if $job->{ready};
7109             }
7110
7111             last if scalar(keys %$jobs) == 0;
7112
7113             if ($readycounter == scalar(keys %$jobs)) {
7114                 print "all '$op' jobs are ready\n";
7115
7116                 # do the complete later (or has already been done)
7117                 last if $completion eq 'skip' || $completion eq 'auto';
7118
7119                 if ($vmiddst && $vmiddst != $vmid) {
7120                     my $agent_running = $qga && qga_check_running($vmid);
7121                     if ($agent_running) {
7122                         print "freeze filesystem\n";
7123                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7124                     } else {
7125                         print "suspend vm\n";
7126                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7127                     }
7128
7129                     # if we clone a disk for a new target vm, we don't switch the disk
7130                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7131
7132                     if ($agent_running) {
7133                         print "unfreeze filesystem\n";
7134                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7135                     } else {
7136                         print "resume vm\n";
7137                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7138                     }
7139
7140                     last;
7141                 } else {
7142
7143                     for my $job_id (sort keys %$jobs) {
7144                         # try to switch the disk if source and destination are on the same guest
7145                         print "$job_id: Completing block job_id...\n";
7146
7147                         my $op;
7148                         if ($completion eq 'complete') {
7149                             $op = 'block-job-complete';
7150                         } elsif ($completion eq 'cancel') {
7151                             $op = 'block-job-cancel';
7152                         } else {
7153                             die "invalid completion value: $completion\n";
7154                         }
7155                         eval { mon_cmd($vmid, $op, device => $job_id) };
7156                         if ($@ =~ m/cannot be completed/) {
7157                             print "$job_id: block job cannot be completed, trying again.\n";
7158                             $err_complete++;
7159                         }else {
7160                             print "$job_id: Completed successfully.\n";
7161                             $jobs->{$job_id}->{complete} = 1;
7162                         }
7163                     }
7164                 }
7165             }
7166             sleep 1;
7167         }
7168     };
7169     my $err = $@;
7170
7171     if ($err) {
7172         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7173         die "block job ($op) error: $err";
7174     }
7175 }
7176
7177 sub qemu_blockjobs_cancel {
7178     my ($vmid, $jobs) = @_;
7179
7180     foreach my $job (keys %$jobs) {
7181         print "$job: Cancelling block job\n";
7182         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7183         $jobs->{$job}->{cancel} = 1;
7184     }
7185
7186     while (1) {
7187         my $stats = mon_cmd($vmid, "query-block-jobs");
7188
7189         my $running_jobs = {};
7190         foreach my $stat (@$stats) {
7191             $running_jobs->{$stat->{device}} = $stat;
7192         }
7193
7194         foreach my $job (keys %$jobs) {
7195
7196             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7197                 print "$job: Done.\n";
7198                 delete $jobs->{$job};
7199             }
7200         }
7201
7202         last if scalar(keys %$jobs) == 0;
7203
7204         sleep 1;
7205     }
7206 }
7207
7208 sub clone_disk {
7209     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7210         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7211
7212     my $newvolid;
7213
7214     if (!$full) {
7215         print "create linked clone of drive $drivename ($drive->{file})\n";
7216         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7217         push @$newvollist, $newvolid;
7218     } else {
7219
7220         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7221         $storeid = $storage if $storage;
7222
7223         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7224
7225         print "create full clone of drive $drivename ($drive->{file})\n";
7226         my $name = undef;
7227         my $size = undef;
7228         if (drive_is_cloudinit($drive)) {
7229             $name = "vm-$newvmid-cloudinit";
7230             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7231             if ($scfg->{path}) {
7232                 $name .= ".$dst_format";
7233             }
7234             $snapname = undef;
7235             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7236         } elsif ($drivename eq 'efidisk0') {
7237             $size = get_efivars_size($conf);
7238         } else {
7239             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7240         }
7241         $newvolid = PVE::Storage::vdisk_alloc(
7242             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7243         );
7244         push @$newvollist, $newvolid;
7245
7246         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7247
7248         if (drive_is_cloudinit($drive)) {
7249             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7250             # if this is the case, we have to complete any block-jobs still there from
7251             # previous drive-mirrors
7252             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7253                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7254             }
7255             goto no_data_clone;
7256         }
7257
7258         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7259         if (!$running || $snapname) {
7260             # TODO: handle bwlimits
7261             if ($drivename eq 'efidisk0') {
7262                 # the relevant data on the efidisk may be smaller than the source
7263                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7264                 # that is given by the OVMF_VARS.fd
7265                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7266                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7267
7268                 # better for Ceph if block size is not too small, see bug #3324
7269                 my $bs = 1024*1024;
7270
7271                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7272                     "if=$src_path", "of=$dst_path"]);
7273             } else {
7274                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7275             }
7276         } else {
7277
7278             my $kvmver = get_running_qemu_version ($vmid);
7279             if (!min_version($kvmver, 2, 7)) {
7280                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7281                     if $drive->{iothread};
7282             }
7283
7284             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7285                 $completion, $qga, $bwlimit);
7286         }
7287     }
7288
7289 no_data_clone:
7290     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7291
7292     my $disk = $drive;
7293     $disk->{format} = undef;
7294     $disk->{file} = $newvolid;
7295     $disk->{size} = $size if defined($size);
7296
7297     return $disk;
7298 }
7299
7300 sub get_running_qemu_version {
7301     my ($vmid) = @_;
7302     my $res = mon_cmd($vmid, "query-version");
7303     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7304 }
7305
7306 sub qemu_use_old_bios_files {
7307     my ($machine_type) = @_;
7308
7309     return if !$machine_type;
7310
7311     my $use_old_bios_files = undef;
7312
7313     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7314         $machine_type = $1;
7315         $use_old_bios_files = 1;
7316     } else {
7317         my $version = extract_version($machine_type, kvm_user_version());
7318         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7319         # load new efi bios files on migration. So this hack is required to allow
7320         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7321         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7322         $use_old_bios_files = !min_version($version, 2, 4);
7323     }
7324
7325     return ($use_old_bios_files, $machine_type);
7326 }
7327
7328 sub get_efivars_size {
7329     my ($conf) = @_;
7330     my $arch = get_vm_arch($conf);
7331     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7332     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7333     return -s $ovmf_vars;
7334 }
7335
7336 sub update_efidisk_size {
7337     my ($conf) = @_;
7338
7339     return if !defined($conf->{efidisk0});
7340
7341     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7342     $disk->{size} = get_efivars_size($conf);
7343     $conf->{efidisk0} = print_drive($disk);
7344
7345     return;
7346 }
7347
7348 sub create_efidisk($$$$$) {
7349     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7350
7351     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7352     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7353
7354     my $vars_size_b = -s $ovmf_vars;
7355     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7356     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7357     PVE::Storage::activate_volumes($storecfg, [$volid]);
7358
7359     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7360     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7361
7362     return ($volid, $size/1024);
7363 }
7364
7365 sub vm_iothreads_list {
7366     my ($vmid) = @_;
7367
7368     my $res = mon_cmd($vmid, 'query-iothreads');
7369
7370     my $iothreads = {};
7371     foreach my $iothread (@$res) {
7372         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7373     }
7374
7375     return $iothreads;
7376 }
7377
7378 sub scsihw_infos {
7379     my ($conf, $drive) = @_;
7380
7381     my $maxdev = 0;
7382
7383     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7384         $maxdev = 7;
7385     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7386         $maxdev = 1;
7387     } else {
7388         $maxdev = 256;
7389     }
7390
7391     my $controller = int($drive->{index} / $maxdev);
7392     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7393         ? "virtioscsi"
7394         : "scsihw";
7395
7396     return ($maxdev, $controller, $controller_prefix);
7397 }
7398
7399 sub windows_version {
7400     my ($ostype) = @_;
7401
7402     return 0 if !$ostype;
7403
7404     my $winversion = 0;
7405
7406     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7407         $winversion = 5;
7408     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7409         $winversion = 6;
7410     } elsif ($ostype =~ m/^win(\d+)$/) {
7411         $winversion = $1;
7412     }
7413
7414     return $winversion;
7415 }
7416
7417 sub resolve_dst_disk_format {
7418         my ($storecfg, $storeid, $src_volname, $format) = @_;
7419         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7420
7421         if (!$format) {
7422             # if no target format is specified, use the source disk format as hint
7423             if ($src_volname) {
7424                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7425                 $format = qemu_img_format($scfg, $src_volname);
7426             } else {
7427                 return $defFormat;
7428             }
7429         }
7430
7431         # test if requested format is supported - else use default
7432         my $supported = grep { $_ eq $format } @$validFormats;
7433         $format = $defFormat if !$supported;
7434         return $format;
7435 }
7436
7437 # NOTE: if this logic changes, please update docs & possibly gui logic
7438 sub find_vmstate_storage {
7439     my ($conf, $storecfg) = @_;
7440
7441     # first, return storage from conf if set
7442     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7443
7444     my ($target, $shared, $local);
7445
7446     foreach_storage_used_by_vm($conf, sub {
7447         my ($sid) = @_;
7448         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7449         my $dst = $scfg->{shared} ? \$shared : \$local;
7450         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7451     });
7452
7453     # second, use shared storage where VM has at least one disk
7454     # third, use local storage where VM has at least one disk
7455     # fall back to local storage
7456     $target = $shared // $local // 'local';
7457
7458     return $target;
7459 }
7460
7461 sub generate_uuid {
7462     my ($uuid, $uuid_str);
7463     UUID::generate($uuid);
7464     UUID::unparse($uuid, $uuid_str);
7465     return $uuid_str;
7466 }
7467
7468 sub generate_smbios1_uuid {
7469     return "uuid=".generate_uuid();
7470 }
7471
7472 sub nbd_stop {
7473     my ($vmid) = @_;
7474
7475     mon_cmd($vmid, 'nbd-server-stop');
7476 }
7477
7478 sub create_reboot_request {
7479     my ($vmid) = @_;
7480     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7481         or die "failed to create reboot trigger file: $!\n";
7482     close($fh);
7483 }
7484
7485 sub clear_reboot_request {
7486     my ($vmid) = @_;
7487     my $path = "/run/qemu-server/$vmid.reboot";
7488     my $res = 0;
7489
7490     $res = unlink($path);
7491     die "could not remove reboot request for $vmid: $!"
7492         if !$res && $! != POSIX::ENOENT;
7493
7494     return $res;
7495 }
7496
7497 sub bootorder_from_legacy {
7498     my ($conf, $bootcfg) = @_;
7499
7500     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7501     my $bootindex_hash = {};
7502     my $i = 1;
7503     foreach my $o (split(//, $boot)) {
7504         $bootindex_hash->{$o} = $i*100;
7505         $i++;
7506     }
7507
7508     my $bootorder = {};
7509
7510     PVE::QemuConfig->foreach_volume($conf, sub {
7511         my ($ds, $drive) = @_;
7512
7513         if (drive_is_cdrom ($drive, 1)) {
7514             if ($bootindex_hash->{d}) {
7515                 $bootorder->{$ds} = $bootindex_hash->{d};
7516                 $bootindex_hash->{d} += 1;
7517             }
7518         } elsif ($bootindex_hash->{c}) {
7519             $bootorder->{$ds} = $bootindex_hash->{c}
7520                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7521             $bootindex_hash->{c} += 1;
7522         }
7523     });
7524
7525     if ($bootindex_hash->{n}) {
7526         for (my $i = 0; $i < $MAX_NETS; $i++) {
7527             my $netname = "net$i";
7528             next if !$conf->{$netname};
7529             $bootorder->{$netname} = $bootindex_hash->{n};
7530             $bootindex_hash->{n} += 1;
7531         }
7532     }
7533
7534     return $bootorder;
7535 }
7536
7537 # Generate default device list for 'boot: order=' property. Matches legacy
7538 # default boot order, but with explicit device names. This is important, since
7539 # the fallback for when neither 'order' nor the old format is specified relies
7540 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7541 sub get_default_bootdevices {
7542     my ($conf) = @_;
7543
7544     my @ret = ();
7545
7546     # harddisk
7547     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7548     push @ret, $first if $first;
7549
7550     # cdrom
7551     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7552     push @ret, $first if $first;
7553
7554     # network
7555     for (my $i = 0; $i < $MAX_NETS; $i++) {
7556         my $netname = "net$i";
7557         next if !$conf->{$netname};
7558         push @ret, $netname;
7559         last;
7560     }
7561
7562     return \@ret;
7563 }
7564
7565 sub device_bootorder {
7566     my ($conf) = @_;
7567
7568     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7569
7570     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7571
7572     my $bootorder = {};
7573     if (!defined($boot) || $boot->{legacy}) {
7574         $bootorder = bootorder_from_legacy($conf, $boot);
7575     } elsif ($boot->{order}) {
7576         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7577         for my $dev (PVE::Tools::split_list($boot->{order})) {
7578             $bootorder->{$dev} = $i++;
7579         }
7580     }
7581
7582     return $bootorder;
7583 }
7584
7585 sub register_qmeventd_handle {
7586     my ($vmid) = @_;
7587
7588     my $fh;
7589     my $peer = "/var/run/qmeventd.sock";
7590     my $count = 0;
7591
7592     for (;;) {
7593         $count++;
7594         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7595         last if $fh;
7596         if ($! != EINTR && $! != EAGAIN) {
7597             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7598         }
7599         if ($count > 4) {
7600             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7601               . "after $count retries\n";
7602         }
7603         usleep(25000);
7604     }
7605
7606     # send handshake to mark VM as backing up
7607     print $fh to_json({vzdump => {vmid => "$vmid"}});
7608
7609     # return handle to be closed later when inhibit is no longer required
7610     return $fh;
7611 }
7612
7613 # bash completion helper
7614
7615 sub complete_backup_archives {
7616     my ($cmdname, $pname, $cvalue) = @_;
7617
7618     my $cfg = PVE::Storage::config();
7619
7620     my $storeid;
7621
7622     if ($cvalue =~ m/^([^:]+):/) {
7623         $storeid = $1;
7624     }
7625
7626     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7627
7628     my $res = [];
7629     foreach my $id (keys %$data) {
7630         foreach my $item (@{$data->{$id}}) {
7631             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7632             push @$res, $item->{volid} if defined($item->{volid});
7633         }
7634     }
7635
7636     return $res;
7637 }
7638
7639 my $complete_vmid_full = sub {
7640     my ($running) = @_;
7641
7642     my $idlist = vmstatus();
7643
7644     my $res = [];
7645
7646     foreach my $id (keys %$idlist) {
7647         my $d = $idlist->{$id};
7648         if (defined($running)) {
7649             next if $d->{template};
7650             next if $running && $d->{status} ne 'running';
7651             next if !$running && $d->{status} eq 'running';
7652         }
7653         push @$res, $id;
7654
7655     }
7656     return $res;
7657 };
7658
7659 sub complete_vmid {
7660     return &$complete_vmid_full();
7661 }
7662
7663 sub complete_vmid_stopped {
7664     return &$complete_vmid_full(0);
7665 }
7666
7667 sub complete_vmid_running {
7668     return &$complete_vmid_full(1);
7669 }
7670
7671 sub complete_storage {
7672
7673     my $cfg = PVE::Storage::config();
7674     my $ids = $cfg->{ids};
7675
7676     my $res = [];
7677     foreach my $sid (keys %$ids) {
7678         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7679         next if !$ids->{$sid}->{content}->{images};
7680         push @$res, $sid;
7681     }
7682
7683     return $res;
7684 }
7685
7686 sub complete_migration_storage {
7687     my ($cmd, $param, $current_value, $all_args) = @_;
7688
7689     my $targetnode = @$all_args[1];
7690
7691     my $cfg = PVE::Storage::config();
7692     my $ids = $cfg->{ids};
7693
7694     my $res = [];
7695     foreach my $sid (keys %$ids) {
7696         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7697         next if !$ids->{$sid}->{content}->{images};
7698         push @$res, $sid;
7699     }
7700
7701     return $res;
7702 }
7703
7704 sub vm_is_paused {
7705     my ($vmid) = @_;
7706     my $qmpstatus = eval {
7707         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7708         mon_cmd($vmid, "query-status");
7709     };
7710     warn "$@\n" if $@;
7711     return $qmpstatus && $qmpstatus->{status} eq "paused";
7712 }
7713
7714 1;