PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 369     },
 370     ostype => {
 371         optional => 1,
 372         type => 'string',
 373         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 374         description => "Specify guest operating system.",
 375         verbose_description => <<EODESC,
 376 Specify guest operating system. This is used to enable special
 377 optimization/features for specific operating systems:
 378
 379 [horizontal]
 380 other;; unspecified OS
 381 wxp;; Microsoft Windows XP
 382 w2k;; Microsoft Windows 2000
 383 w2k3;; Microsoft Windows 2003
 384 w2k8;; Microsoft Windows 2008
 385 wvista;; Microsoft Windows Vista
 386 win7;; Microsoft Windows 7
 387 win8;; Microsoft Windows 8/2012/2012r2
 388 win10;; Microsoft Windows 10/2016/2019
 389 l24;; Linux 2.4 Kernel
 390 l26;; Linux 2.6 - 5.X Kernel
 391 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 392 EODESC
 393     },
 394     boot => {
 395         optional => 1,
 396         type => 'string', format => 'pve-qm-boot',
 397         description => "Specify guest boot order. Use with 'order=', usage with"
 398                      . " no key or 'legacy=' is deprecated.",
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     keephugepages => {
 440         optional => 1,
 441         type => 'boolean',
 442         default => 0,
 443         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 444             ." after VM shutdown and can be used for subsequent starts.",
 445     },
 446     vcpus => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "Number of hotplugged vcpus.",
 450         minimum => 1,
 451         default => 0,
 452     },
 453     acpi => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Enable/disable ACPI.",
 457         default => 1,
 458     },
 459     agent => {
 460         optional => 1,
 461         description => "Enable/disable Qemu GuestAgent and its properties.",
 462         type => 'string',
 463         format => $agent_fmt,
 464     },
 465     kvm => {
 466         optional => 1,
 467         type => 'boolean',
 468         description => "Enable/disable KVM hardware virtualization.",
 469         default => 1,
 470     },
 471     tdf => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable time drift fix.",
 475         default => 0,
 476     },
 477     localtime => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Set the real time clock to local time. This is enabled by default if ostype"
 481             ." indicates a Microsoft OS.",
 482     },
 483     freeze => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 487     },
 488     vga => {
 489         optional => 1,
 490         type => 'string', format => $vga_fmt,
 491         description => "Configure the VGA hardware.",
 492         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 493             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 494             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 495             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 496             ." display server. For win* OS you can select how many independent displays you want,"
 497             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 498             ." using a serial device as terminal.",
 499     },
 500     watchdog => {
 501         optional => 1,
 502         type => 'string', format => 'pve-qm-watchdog',
 503         description => "Create a virtual hardware watchdog device.",
 504         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 505             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 506             ." else the watchdog will reset the guest (or execute the respective action specified)",
 507     },
 508     startdate => {
 509         optional => 1,
 510         type => 'string',
 511         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 512         description => "Set the initial date of the real time clock. Valid format for date are:"
 513             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 514         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 515         default => 'now',
 516     },
 517     startup =>  get_standard_option('pve-startup-order'),
 518     template => {
 519         optional => 1,
 520         type => 'boolean',
 521         description => "Enable/disable Template.",
 522         default => 0,
 523     },
 524     args => {
 525         optional => 1,
 526         type => 'string',
 527         description => "Arbitrary arguments passed to kvm.",
 528         verbose_description => <<EODESCR,
 529 Arbitrary arguments passed to kvm, for example:
 530
 531 args: -no-reboot -no-hpet
 532
 533 NOTE: this option is for experts only.
 534 EODESCR
 535     },
 536     tablet => {
 537         optional => 1,
 538         type => 'boolean',
 539         default => 1,
 540         description => "Enable/disable the USB tablet device.",
 541         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 542             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 543             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 544             ." may consider disabling this to save some context switches. This is turned off by"
 545             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 546     },
 547     migrate_speed => {
 548         optional => 1,
 549         type => 'integer',
 550         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 551         minimum => 0,
 552         default => 0,
 553     },
 554     migrate_downtime => {
 555         optional => 1,
 556         type => 'number',
 557         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 558         minimum => 0,
 559         default => 0.1,
 560     },
 561     cdrom => {
 562         optional => 1,
 563         type => 'string', format => 'pve-qm-ide',
 564         typetext => '<volume>',
 565         description => "This is an alias for option -ide2",
 566     },
 567     cpu => {
 568         optional => 1,
 569         description => "Emulated CPU type.",
 570         type => 'string',
 571         format => 'pve-vm-cpu-conf',
 572     },
 573     parent => get_standard_option('pve-snapshot-name', {
 574         optional => 1,
 575         description => "Parent snapshot name. This is used internally, and should not be modified.",
 576     }),
 577     snaptime => {
 578         optional => 1,
 579         description => "Timestamp for snapshots.",
 580         type => 'integer',
 581         minimum => 0,
 582     },
 583     vmstate => {
 584         optional => 1,
 585         type => 'string', format => 'pve-volume-id',
 586         description => "Reference to a volume which stores the VM state. This is used internally"
 587             ." for snapshots.",
 588     },
 589     vmstatestorage => get_standard_option('pve-storage-id', {
 590         description => "Default storage for VM state volumes/files.",
 591         optional => 1,
 592     }),
 593     runningmachine => get_standard_option('pve-qemu-machine', {
 594         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 595             ." for snapshots.",
 596     }),
 597     runningcpu => {
 598         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 599             ." internally for snapshots.",
 600         optional => 1,
 601         type => 'string',
 602         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 603         format_description => 'QEMU -cpu parameter'
 604     },
 605     machine => get_standard_option('pve-qemu-machine'),
 606     arch => {
 607         description => "Virtual processor architecture. Defaults to the host.",
 608         optional => 1,
 609         type => 'string',
 610         enum => [qw(x86_64 aarch64)],
 611     },
 612     smbios1 => {
 613         description => "Specify SMBIOS type 1 fields.",
 614         type => 'string', format => 'pve-qm-smbios1',
 615         maxLength => 512,
 616         optional => 1,
 617     },
 618     protection => {
 619         optional => 1,
 620         type => 'boolean',
 621         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 622             ." remove disk operations.",
 623         default => 0,
 624     },
 625     bios => {
 626         optional => 1,
 627         type => 'string',
 628         enum => [ qw(seabios ovmf) ],
 629         description => "Select BIOS implementation.",
 630         default => 'seabios',
 631     },
 632     vmgenid => {
 633         type => 'string',
 634         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 635         format_description => 'UUID',
 636         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 637             ." to disable explicitly.",
 638         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 639             ." value identifier to the guest OS. This allows to notify the guest operating system"
 640             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 641             ." execution or creation from a template). The guest operating system notices the"
 642             ." change, and is then able to react as appropriate by marking its copies of"
 643             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 644             ."Note that auto-creation only works when done through API/CLI create or update methods"
 645             .", but not when manually editing the config file.",
 646         default => "1 (autogenerated)",
 647         optional => 1,
 648     },
 649     hookscript => {
 650         type => 'string',
 651         format => 'pve-volume-id',
 652         optional => 1,
 653         description => "Script that will be executed during various steps in the vms lifetime.",
 654     },
 655     ivshmem => {
 656         type => 'string',
 657         format => $ivshmem_fmt,
 658         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 659             ." the host.",
 660         optional => 1,
 661     },
 662     audio0 => {
 663         type => 'string',
 664         format => $audio_fmt,
 665         description => "Configure a audio device, useful in combination with QXL/Spice.",
 666         optional => 1
 667     },
 668     spice_enhancements => {
 669         type => 'string',
 670         format => $spice_enhancements_fmt,
 671         description => "Configure additional enhancements for SPICE.",
 672         optional => 1
 673     },
 674     tags => {
 675         type => 'string', format => 'pve-tag-list',
 676         description => 'Tags of the VM. This is only meta information.',
 677         optional => 1,
 678     },
 679     rng0 => {
 680         type => 'string',
 681         format => $rng_fmt,
 682         description => "Configure a VirtIO-based Random Number Generator.",
 683         optional => 1,
 684     },
 685 };
 686
 687 my $cicustom_fmt = {
 688     meta => {
 689         type => 'string',
 690         optional => 1,
 691         description => 'Specify a custom file containing all meta data passed to the VM via"
 692             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 693         format => 'pve-volume-id',
 694         format_description => 'volume',
 695     },
 696     network => {
 697         type => 'string',
 698         optional => 1,
 699         description => 'Specify a custom file containing all network data passed to the VM via'
 700             .' cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704     user => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all user data passed to the VM via'
 708             .' cloud-init.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712 };
 713 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 714
 715 my $confdesc_cloudinit = {
 716     citype => {
 717         optional => 1,
 718         type => 'string',
 719         description => 'Specifies the cloud-init configuration format. The default depends on the'
 720             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 721             .' and `configdrive2` for windows.',
 722         enum => ['configdrive2', 'nocloud', 'opennebula'],
 723     },
 724     ciuser => {
 725         optional => 1,
 726         type => 'string',
 727         description => "cloud-init: User name to change ssh keys and password for instead of the"
 728             ." image's configured default user.",
 729     },
 730     cipassword => {
 731         optional => 1,
 732         type => 'string',
 733         description => 'cloud-init: Password to assign the user. Using this is generally not'
 734             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 735             .' support hashed passwords.',
 736     },
 737     cicustom => {
 738         optional => 1,
 739         type => 'string',
 740         description => 'cloud-init: Specify custom files to replace the automatically generated'
 741             .' ones at start.',
 742         format => 'pve-qm-cicustom',
 743     },
 744     searchdomain => {
 745         optional => 1,
 746         type => 'string',
 747         description => "cloud-init: Sets DNS search domains for a container. Create will'
 748             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 749             .' are set.",
 750     },
 751     nameserver => {
 752         optional => 1,
 753         type => 'string', format => 'address-list',
 754         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 755             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 756             .' are set.",
 757     },
 758     sshkeys => {
 759         optional => 1,
 760         type => 'string',
 761         format => 'urlencoded',
 762         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 763     },
 764 };
 765
 766 # what about other qemu settings ?
 767 #cpu => 'string',
 768 #machine => 'string',
 769 #fda => 'file',
 770 #fdb => 'file',
 771 #mtdblock => 'file',
 772 #sd => 'file',
 773 #pflash => 'file',
 774 #snapshot => 'bool',
 775 #bootp => 'file',
 776 ##tftp => 'dir',
 777 ##smb => 'dir',
 778 #kernel => 'file',
 779 #append => 'string',
 780 #initrd => 'file',
 781 ##soundhw => 'string',
 782
 783 while (my ($k, $v) = each %$confdesc) {
 784     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 785 }
 786
 787 my $MAX_USB_DEVICES = 5;
 788 my $MAX_NETS = 32;
 789 my $MAX_SERIAL_PORTS = 4;
 790 my $MAX_PARALLEL_PORTS = 3;
 791 my $MAX_NUMA = 8;
 792
 793 my $numa_fmt = {
 794     cpus => {
 795         type => "string",
 796         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 797         description => "CPUs accessing this NUMA node.",
 798         format_description => "id[-id];...",
 799     },
 800     memory => {
 801         type => "number",
 802         description => "Amount of memory this NUMA node provides.",
 803         optional => 1,
 804     },
 805     hostnodes => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "Host NUMA nodes to use.",
 809         format_description => "id[-id];...",
 810         optional => 1,
 811     },
 812     policy => {
 813         type => 'string',
 814         enum => [qw(preferred bind interleave)],
 815         description => "NUMA allocation policy.",
 816         optional => 1,
 817     },
 818 };
 819 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 820 my $numadesc = {
 821     optional => 1,
 822     type => 'string', format => $numa_fmt,
 823     description => "NUMA topology.",
 824 };
 825 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 826
 827 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 828     $confdesc->{"numa$i"} = $numadesc;
 829 }
 830
 831 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 832                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 833                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 834 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 835
 836 my $net_fmt_bridge_descr = <<__EOD__;
 837 Bridge to attach the network device to. The Proxmox VE standard bridge
 838 is called 'vmbr0'.
 839
 840 If you do not specify a bridge, we create a kvm user (NATed) network
 841 device, which provides DHCP and DNS services. The following addresses
 842 are used:
 843
 844  10.0.2.2   Gateway
 845  10.0.2.3   DNS Server
 846  10.0.2.4   SMB Server
 847
 848 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 849 __EOD__
 850
 851 my $net_fmt = {
 852     macaddr  => get_standard_option('mac-addr', {
 853         description => "MAC address. That address must be unique withing your network. This is"
 854             ." automatically generated if not specified.",
 855     }),
 856     model => {
 857         type => 'string',
 858         description => "Network Card Model. The 'virtio' model provides the best performance with"
 859             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 860             ." best to use 'e1000'.",
 861         enum => $nic_model_list,
 862         default_key => 1,
 863     },
 864     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 865     bridge => {
 866         type => 'string',
 867         description => $net_fmt_bridge_descr,
 868         format_description => 'bridge',
 869         pattern => '[-_.\w\d]+',
 870         optional => 1,
 871     },
 872     queues => {
 873         type => 'integer',
 874         minimum => 0, maximum => 16,
 875         description => 'Number of packet queues to be used on the device.',
 876         optional => 1,
 877     },
 878     rate => {
 879         type => 'number',
 880         minimum => 0,
 881         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 882         optional => 1,
 883     },
 884     tag => {
 885         type => 'integer',
 886         minimum => 1, maximum => 4094,
 887         description => 'VLAN tag to apply to packets on this interface.',
 888         optional => 1,
 889     },
 890     trunks => {
 891         type => 'string',
 892         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 893         description => 'VLAN trunks to pass through this interface.',
 894         format_description => 'vlanid[;vlanid...]',
 895         optional => 1,
 896     },
 897     firewall => {
 898         type => 'boolean',
 899         description => 'Whether this interface should be protected by the firewall.',
 900         optional => 1,
 901     },
 902     link_down => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be disconnected (like pulling the plug).',
 905         optional => 1,
 906     },
 907     mtu => {
 908         type => 'integer',
 909         minimum => 1, maximum => 65520,
 910         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 911         optional => 1,
 912     },
 913 };
 914
 915 my $netdesc = {
 916     optional => 1,
 917     type => 'string', format => $net_fmt,
 918     description => "Specify network devices.",
 919 };
 920
 921 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 922
 923 my $ipconfig_fmt = {
 924     ip => {
 925         type => 'string',
 926         format => 'pve-ipv4-config',
 927         format_description => 'IPv4Format/CIDR',
 928         description => 'IPv4 address in CIDR format.',
 929         optional => 1,
 930         default => 'dhcp',
 931     },
 932     gw => {
 933         type => 'string',
 934         format => 'ipv4',
 935         format_description => 'GatewayIPv4',
 936         description => 'Default gateway for IPv4 traffic.',
 937         optional => 1,
 938         requires => 'ip',
 939     },
 940     ip6 => {
 941         type => 'string',
 942         format => 'pve-ipv6-config',
 943         format_description => 'IPv6Format/CIDR',
 944         description => 'IPv6 address in CIDR format.',
 945         optional => 1,
 946         default => 'dhcp',
 947     },
 948     gw6 => {
 949         type => 'string',
 950         format => 'ipv6',
 951         format_description => 'GatewayIPv6',
 952         description => 'Default gateway for IPv6 traffic.',
 953         optional => 1,
 954         requires => 'ip6',
 955     },
 956 };
 957 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 958 my $ipconfigdesc = {
 959     optional => 1,
 960     type => 'string', format => 'pve-qm-ipconfig',
 961     description => <<'EODESCR',
 962 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 963
 964 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 965
 966 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 967 gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 969 cloud-init 19.4 or newer.
 970
 971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 972 dhcp on IPv4.
 973 EODESCR
 974 };
 975 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 976
 977 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 978     $confdesc->{"net$i"} = $netdesc;
 979     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 980 }
 981
 982 foreach my $key (keys %$confdesc_cloudinit) {
 983     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 984 }
 985
 986 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 987 sub verify_volume_id_or_qm_path {
 988     my ($volid, $noerr) = @_;
 989
 990     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 991         return $volid;
 992     }
 993
 994     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 995     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 996     if ($@) {
 997         return if $noerr;
 998         die $@;
 999     }
1000     return $volid;
1001 }
1002
1003 my $usb_fmt = {
1004     host => {
1005         default_key => 1,
1006         type => 'string', format => 'pve-qm-usb-device',
1007         format_description => 'HOSTUSBDEVICE|spice',
1008         description => <<EODESCR,
1009 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1010
1011  'bus-port(.port)*' (decimal numbers) or
1012  'vendor_id:product_id' (hexadeciaml numbers) or
1013  'spice'
1014
1015 You can use the 'lsusb -t' command to list existing usb devices.
1016
1017 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1018 machines - use with special care.
1019
1020 The value 'spice' can be used to add a usb redirection devices for spice.
1021 EODESCR
1022     },
1023     usb3 => {
1024         optional => 1,
1025         type => 'boolean',
1026         description => "Specifies whether if given host option is a USB3 device or port.",
1027         default => 0,
1028     },
1029 };
1030
1031 my $usbdesc = {
1032     optional => 1,
1033     type => 'string', format => $usb_fmt,
1034     description => "Configure an USB device (n is 0 to 4).",
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1037
1038 my $serialdesc = {
1039         optional => 1,
1040         type => 'string',
1041         pattern => '(/dev/.+|socket)',
1042         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1043         verbose_description =>  <<EODESCR,
1044 Create a serial device inside the VM (n is 0 to 3), and pass through a
1045 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046 host side (use 'qm terminal' to open a terminal connection).
1047
1048 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1049 use with special care.
1050
1051 CAUTION: Experimental! User reported problems with this option.
1052 EODESCR
1053 };
1054
1055 my $paralleldesc= {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1059         description =>  "Map host parallel devices (n is 0 to 2).",
1060         verbose_description =>  <<EODESCR,
1061 Map host parallel devices (n is 0 to 2).
1062
1063 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1064 machines - use with special care.
1065
1066 CAUTION: Experimental! User reported problems with this option.
1067 EODESCR
1068 };
1069
1070 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1071     $confdesc->{"parallel$i"} = $paralleldesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1075     $confdesc->{"serial$i"} = $serialdesc;
1076 }
1077
1078 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1079     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1080 }
1081
1082 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1083     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1084 }
1085
1086 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1087     $confdesc->{"usb$i"} = $usbdesc;
1088 }
1089
1090 my $boot_fmt = {
1091     legacy => {
1092         optional => 1,
1093         default_key => 1,
1094         type => 'string',
1095         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1096                      . " Deprecated, use 'order=' instead.",
1097         pattern => '[acdn]{1,4}',
1098         format_description => "[acdn]{1,4}",
1099
1100         # note: this is also the fallback if boot: is not given at all
1101         default => 'cdn',
1102     },
1103     order => {
1104         optional => 1,
1105         type => 'string',
1106         format => 'pve-qm-bootdev-list',
1107         format_description => "device[;device...]",
1108         description => <<EODESC,
1109 The guest will attempt to boot from devices in the order they appear here.
1110
1111 Disks, optical drives and passed-through storage USB devices will be directly
1112 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1113 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1114
1115 Note that only devices in this list will be marked as bootable and thus loaded
1116 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1117 (e.g. software-raid), you need to specify all of them here.
1118
1119 Overrides the deprecated 'legacy=[acdn]*' value when given.
1120 EODESC
1121     },
1122 };
1123 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1124
1125 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1126 sub verify_bootdev {
1127     my ($dev, $noerr) = @_;
1128
1129     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1130
1131     my $check = sub {
1132         my ($base) = @_;
1133         return 0 if $dev !~ m/^$base\d+$/;
1134         return 0 if !$confdesc->{$dev};
1135         return 1;
1136     };
1137
1138     return $dev if $check->("net");
1139     return $dev if $check->("usb");
1140     return $dev if $check->("hostpci");
1141
1142     return if $noerr;
1143     die "invalid boot device '$dev'\n";
1144 }
1145
1146 sub print_bootorder {
1147     my ($devs) = @_;
1148     return "" if !@$devs;
1149     my $data = { order => join(';', @$devs) };
1150     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1151 }
1152
1153 my $kvm_api_version = 0;
1154
1155 sub kvm_version {
1156     return $kvm_api_version if $kvm_api_version;
1157
1158     open my $fh, '<', '/dev/kvm' or return;
1159
1160     # 0xae00 => KVM_GET_API_VERSION
1161     $kvm_api_version = ioctl($fh, 0xae00, 0);
1162     close($fh);
1163
1164     return $kvm_api_version;
1165 }
1166
1167 my $kvm_user_version = {};
1168 my $kvm_mtime = {};
1169
1170 sub kvm_user_version {
1171     my ($binary) = @_;
1172
1173     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1174     my $st = stat($binary);
1175
1176     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1177     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1178         $cachedmtime == $st->mtime;
1179
1180     $kvm_user_version->{$binary} = 'unknown';
1181     $kvm_mtime->{$binary} = $st->mtime;
1182
1183     my $code = sub {
1184         my $line = shift;
1185         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186             $kvm_user_version->{$binary} = $2;
1187         }
1188     };
1189
1190     eval { run_command([$binary, '--version'], outfunc => $code); };
1191     warn $@ if $@;
1192
1193     return $kvm_user_version->{$binary};
1194
1195 }
1196 my sub extract_version {
1197     my ($machine_type, $version) = @_;
1198     $version = kvm_user_version() if !defined($version);
1199     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1200 }
1201
1202 sub kernel_has_vhost_net {
1203     return -c '/dev/vhost-net';
1204 }
1205
1206 sub option_exists {
1207     my $key = shift;
1208     return defined($confdesc->{$key});
1209 }
1210
1211 my $cdrom_path;
1212 sub get_cdrom_path {
1213
1214     return  $cdrom_path if $cdrom_path;
1215
1216     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1217     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1218     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1219 }
1220
1221 sub get_iso_path {
1222     my ($storecfg, $vmid, $cdrom) = @_;
1223
1224     if ($cdrom eq 'cdrom') {
1225         return get_cdrom_path();
1226     } elsif ($cdrom eq 'none') {
1227         return '';
1228     } elsif ($cdrom =~ m|^/|) {
1229         return $cdrom;
1230     } else {
1231         return PVE::Storage::path($storecfg, $cdrom);
1232     }
1233 }
1234
1235 # try to convert old style file names to volume IDs
1236 sub filename_to_volume_id {
1237     my ($vmid, $file, $media) = @_;
1238
1239      if (!($file eq 'none' || $file eq 'cdrom' ||
1240           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1241
1242         return if $file =~ m|/|;
1243
1244         if ($media && $media eq 'cdrom') {
1245             $file = "local:iso/$file";
1246         } else {
1247             $file = "local:$vmid/$file";
1248         }
1249     }
1250
1251     return $file;
1252 }
1253
1254 sub verify_media_type {
1255     my ($opt, $vtype, $media) = @_;
1256
1257     return if !$media;
1258
1259     my $etype;
1260     if ($media eq 'disk') {
1261         $etype = 'images';
1262     } elsif ($media eq 'cdrom') {
1263         $etype = 'iso';
1264     } else {
1265         die "internal error";
1266     }
1267
1268     return if ($vtype eq $etype);
1269
1270     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1271 }
1272
1273 sub cleanup_drive_path {
1274     my ($opt, $storecfg, $drive) = @_;
1275
1276     # try to convert filesystem paths to volume IDs
1277
1278     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1279         ($drive->{file} !~ m|^/dev/.+|) &&
1280         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1281         ($drive->{file} !~ m/^\d+$/)) {
1282         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1283         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1284             if !$vtype;
1285         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1286         verify_media_type($opt, $vtype, $drive->{media});
1287         $drive->{file} = $volid;
1288     }
1289
1290     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1291 }
1292
1293 sub parse_hotplug_features {
1294     my ($data) = @_;
1295
1296     my $res = {};
1297
1298     return $res if $data eq '0';
1299
1300     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1301
1302     foreach my $feature (PVE::Tools::split_list($data)) {
1303         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304             $res->{$1} = 1;
1305         } else {
1306             die "invalid hotplug feature '$feature'\n";
1307         }
1308     }
1309     return $res;
1310 }
1311
1312 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1313 sub pve_verify_hotplug_features {
1314     my ($value, $noerr) = @_;
1315
1316     return $value if parse_hotplug_features($value);
1317
1318     return if $noerr;
1319
1320     die "unable to parse hotplug option\n";
1321 }
1322
1323 sub scsi_inquiry {
1324     my($fh, $noerr) = @_;
1325
1326     my $SG_IO = 0x2285;
1327     my $SG_GET_VERSION_NUM = 0x2282;
1328
1329     my $versionbuf = "\x00" x 8;
1330     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1331     if (!$ret) {
1332         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333         return;
1334     }
1335     my $version = unpack("I", $versionbuf);
1336     if ($version < 30000) {
1337         die "scsi generic interface too old\n"  if !$noerr;
1338         return;
1339     }
1340
1341     my $buf = "\x00" x 36;
1342     my $sensebuf = "\x00" x 8;
1343     my $cmd = pack("C x3 C x1", 0x12, 36);
1344
1345     # see /usr/include/scsi/sg.h
1346     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1347
1348     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1349                       length($sensebuf), 0, length($buf), $buf,
1350                       $cmd, $sensebuf, 6000);
1351
1352     $ret = ioctl($fh, $SG_IO, $packet);
1353     if (!$ret) {
1354         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1355         return;
1356     }
1357
1358     my @res = unpack($sg_io_hdr_t, $packet);
1359     if ($res[17] || $res[18]) {
1360         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1361         return;
1362     }
1363
1364     my $res = {};
1365     (my $byte0, my $byte1, $res->{vendor},
1366      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1367
1368     $res->{removable} = $byte1 & 128 ? 1 : 0;
1369     $res->{type} = $byte0 & 31;
1370
1371     return $res;
1372 }
1373
1374 sub path_is_scsi {
1375     my ($path) = @_;
1376
1377     my $fh = IO::File->new("+<$path") || return;
1378     my $res = scsi_inquiry($fh, 1);
1379     close($fh);
1380
1381     return $res;
1382 }
1383
1384 sub print_tabletdevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1388
1389     # we use uhci for old VMs because tablet driver was buggy in older qemu
1390     my $usbbus;
1391     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1392         $usbbus = 'ehci';
1393     } else {
1394         $usbbus = 'uhci';
1395     }
1396
1397     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 }
1399
1400 sub print_keyboarddevice_full {
1401     my ($conf, $arch, $machine) = @_;
1402
1403     return if $arch ne 'aarch64';
1404
1405     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 }
1407
1408 my sub get_drive_id {
1409     my ($drive) = @_;
1410     return "$drive->{interface}$drive->{index}";
1411 }
1412
1413 sub print_drivedevice_full {
1414     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1415
1416     my $device = '';
1417     my $maxdev = 0;
1418
1419     my $drive_id = get_drive_id($drive);
1420     if ($drive->{interface} eq 'virtio') {
1421         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1422         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1423         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1424     } elsif ($drive->{interface} eq 'scsi') {
1425
1426         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1427         my $unit = $drive->{index} % $maxdev;
1428         my $devicetype = 'hd';
1429         my $path = '';
1430         if (drive_is_cdrom($drive)) {
1431             $devicetype = 'cd';
1432         } else {
1433             if ($drive->{file} =~ m|^/|) {
1434                 $path = $drive->{file};
1435                 if (my $info = path_is_scsi($path)) {
1436                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1437                         $devicetype = 'block';
1438                     } elsif ($info->{type} == 1) { # tape
1439                         $devicetype = 'generic';
1440                     }
1441                 }
1442             } else {
1443                  $path = PVE::Storage::path($storecfg, $drive->{file});
1444             }
1445
1446             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1447             my $version = extract_version($machine_type, kvm_user_version());
1448             if ($path =~ m/^iscsi\:\/\// &&
1449                !min_version($version, 4, 1)) {
1450                 $devicetype = 'generic';
1451             }
1452         }
1453
1454         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1455             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1456         } else {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1458                 .",lun=$drive->{index}";
1459         }
1460         $device .= ",drive=drive-$drive_id,id=$drive_id";
1461
1462         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1463             $device .= ",rotation_rate=1";
1464         }
1465         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1466
1467     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1468         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1469         my $controller = int($drive->{index} / $maxdev);
1470         my $unit = $drive->{index} % $maxdev;
1471         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1472
1473         $device = "ide-$devicetype";
1474         if ($drive->{interface} eq 'ide') {
1475             $device .= ",bus=ide.$controller,unit=$unit";
1476         } else {
1477             $device .= ",bus=ahci$controller.$unit";
1478         }
1479         $device .= ",drive=drive-$drive_id,id=$drive_id";
1480
1481         if ($devicetype eq 'hd') {
1482             if (my $model = $drive->{model}) {
1483                 $model = URI::Escape::uri_unescape($model);
1484                 $device .= ",model=$model";
1485             }
1486             if ($drive->{ssd}) {
1487                 $device .= ",rotation_rate=1";
1488             }
1489         }
1490         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1491     } elsif ($drive->{interface} eq 'usb') {
1492         die "implement me";
1493         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1494     } else {
1495         die "unsupported interface type";
1496     }
1497
1498     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1499
1500     if (my $serial = $drive->{serial}) {
1501         $serial = URI::Escape::uri_unescape($serial);
1502         $device .= ",serial=$serial";
1503     }
1504
1505
1506     return $device;
1507 }
1508
1509 sub get_initiator_name {
1510     my $initiator;
1511
1512     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1513     while (defined(my $line = <$fh>)) {
1514         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1515         $initiator = $1;
1516         last;
1517     }
1518     $fh->close();
1519
1520     return $initiator;
1521 }
1522
1523 sub print_drive_commandline_full {
1524     my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1525
1526     my $path;
1527     my $volid = $drive->{file};
1528     my $format = $drive->{format};
1529     my $drive_id = get_drive_id($drive);
1530
1531     if (drive_is_cdrom($drive)) {
1532         $path = get_iso_path($storecfg, $vmid, $volid);
1533         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1534     } else {
1535         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1536         if ($storeid) {
1537             $path = PVE::Storage::path($storecfg, $volid);
1538             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1539             $format //= qemu_img_format($scfg, $volname);
1540         } else {
1541             $path = $volid;
1542             $format //= "raw";
1543         }
1544    }
1545
1546    my $is_rbd = $path =~ m/^rbd:/;
1547
1548     my $opts = '';
1549     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1550     foreach my $o (@qemu_drive_options) {
1551         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1552     }
1553
1554     # snapshot only accepts on|off
1555     if (defined($drive->{snapshot})) {
1556         my $v = $drive->{snapshot} ? 'on' : 'off';
1557         $opts .= ",snapshot=$v";
1558     }
1559
1560     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1561         my ($dir, $qmpname) = @$type;
1562         if (my $v = $drive->{"mbps$dir"}) {
1563             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1564         }
1565         if (my $v = $drive->{"mbps${dir}_max"}) {
1566             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1567         }
1568         if (my $v = $drive->{"bps${dir}_max_length"}) {
1569             $opts .= ",throttling.bps$qmpname-max-length=$v";
1570         }
1571         if (my $v = $drive->{"iops${dir}"}) {
1572             $opts .= ",throttling.iops$qmpname=$v";
1573         }
1574         if (my $v = $drive->{"iops${dir}_max"}) {
1575             $opts .= ",throttling.iops$qmpname-max=$v";
1576         }
1577         if (my $v = $drive->{"iops${dir}_max_length"}) {
1578             $opts .= ",throttling.iops$qmpname-max-length=$v";
1579         }
1580     }
1581
1582     if ($pbs_name) {
1583         $format = "rbd" if $is_rbd;
1584         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1585             if !$format;
1586         $opts .= ",format=alloc-track,file.driver=$format";
1587     } elsif ($format) {
1588         $opts .= ",format=$format";
1589     }
1590
1591     my $cache_direct = 0;
1592
1593     if (my $cache = $drive->{cache}) {
1594         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1595     } elsif (!drive_is_cdrom($drive)) {
1596         $opts .= ",cache=none";
1597         $cache_direct = 1;
1598     }
1599
1600     # aio native works only with O_DIRECT
1601     if (!$drive->{aio}) {
1602         if($cache_direct) {
1603             $opts .= ",aio=native";
1604         } else {
1605             $opts .= ",aio=threads";
1606         }
1607     }
1608
1609     if (!drive_is_cdrom($drive)) {
1610         my $detectzeroes;
1611         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1612             $detectzeroes = 'off';
1613         } elsif ($drive->{discard}) {
1614             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1615         } else {
1616             # This used to be our default with discard not being specified:
1617             $detectzeroes = 'on';
1618         }
1619
1620         # note: 'detect-zeroes' works per blockdev and we want it to persist
1621         # after the alloc-track is removed, so put it on 'file' directly
1622         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1623         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1624     }
1625
1626     if ($pbs_name) {
1627         $opts .= ",backing=$pbs_name";
1628         $opts .= ",auto-remove=on";
1629     }
1630
1631     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1632     my $file_param = "file";
1633     if ($pbs_name) {
1634         # non-rbd drivers require the underlying file to be a seperate block
1635         # node, so add a second .file indirection
1636         $file_param .= ".file" if !$is_rbd;
1637         $file_param .= ".filename";
1638     }
1639     my $pathinfo = $path ? "$file_param=$path," : '';
1640
1641     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1642 }
1643
1644 sub print_pbs_blockdev {
1645     my ($pbs_conf, $pbs_name) = @_;
1646     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1647     $blockdev .= ",repository=$pbs_conf->{repository}";
1648     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1649     $blockdev .= ",archive=$pbs_conf->{archive}";
1650     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1651     return $blockdev;
1652 }
1653
1654 sub print_netdevice_full {
1655     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1656
1657     my $device = $net->{model};
1658     if ($net->{model} eq 'virtio') {
1659          $device = 'virtio-net-pci';
1660      };
1661
1662     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1663     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1664     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1665         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1666         # and out of each queue plus one config interrupt and control vector queue
1667         my $vectors = $net->{queues} * 2 + 2;
1668         $tmpstr .= ",vectors=$vectors,mq=on";
1669     }
1670     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1671
1672     if (my $mtu = $net->{mtu}) {
1673         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1674             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1675             if ($mtu == 1) {
1676                  $mtu = $bridge_mtu;
1677             } elsif ($mtu < 576) {
1678                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1679             } elsif ($mtu > $bridge_mtu) {
1680                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1681             }
1682             $tmpstr .= ",host_mtu=$mtu";
1683         } else {
1684             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1685         }
1686     }
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     if (is_native($arch)) {
1725         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1726     }
1727
1728     my $vmname = $conf->{name} || "vm$vmid";
1729
1730     my $netdev = "";
1731     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733     if ($net->{bridge}) {
1734         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1735             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1736     } else {
1737         $netdev = "type=user,id=$netid,hostname=$vmname";
1738     }
1739
1740     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1741
1742     return $netdev;
1743 }
1744
1745 my $vga_map = {
1746     'cirrus' => 'cirrus-vga',
1747     'std' => 'VGA',
1748     'vmware' => 'vmware-svga',
1749     'virtio' => 'virtio-vga',
1750 };
1751
1752 sub print_vga_device {
1753     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1754
1755     my $type = $vga_map->{$vga->{type}};
1756     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1757         $type = 'virtio-gpu';
1758     }
1759     my $vgamem_mb = $vga->{memory};
1760
1761     my $max_outputs = '';
1762     if ($qxlnum) {
1763         $type = $id ? 'qxl' : 'qxl-vga';
1764
1765         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1766             # set max outputs so linux can have up to 4 qxl displays with one device
1767             if (min_version($machine_version, 4, 1)) {
1768                 $max_outputs = ",max_outputs=4";
1769             }
1770         }
1771     }
1772
1773     die "no devicetype for $vga->{type}\n" if !$type;
1774
1775     my $memory = "";
1776     if ($vgamem_mb) {
1777         if ($vga->{type} eq 'virtio') {
1778             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1779             $memory = ",max_hostmem=$bytes";
1780         } elsif ($qxlnum) {
1781             # from https://www.spice-space.org/multiple-monitors.html
1782             $memory = ",vgamem_mb=$vga->{memory}";
1783             my $ram = $vgamem_mb * 4;
1784             my $vram = $vgamem_mb * 2;
1785             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1786         } else {
1787             $memory = ",vgamem_mb=$vga->{memory}";
1788         }
1789     } elsif ($qxlnum && $id) {
1790         $memory = ",ram_size=67108864,vram_size=33554432";
1791     }
1792
1793     my $edidoff = "";
1794     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1795         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1796     }
1797
1798     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1799     my $vgaid = "vga" . ($id // '');
1800     my $pciaddr;
1801
1802     if ($q35 && $vgaid eq 'vga') {
1803         # the first display uses pcie.0 bus on q35 machines
1804         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1805     } else {
1806         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1807     }
1808
1809     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1810 }
1811
1812 sub parse_number_sets {
1813     my ($set) = @_;
1814     my $res = [];
1815     foreach my $part (split(/;/, $set)) {
1816         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818             push @$res, [ $1, $2 ];
1819         } else {
1820             die "invalid range: $part\n";
1821         }
1822     }
1823     return $res;
1824 }
1825
1826 sub parse_numa {
1827     my ($data) = @_;
1828
1829     my $res = parse_property_string($numa_fmt, $data);
1830     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1831     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1832     return $res;
1833 }
1834
1835 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1836 sub parse_net {
1837     my ($data) = @_;
1838
1839     my $res = eval { parse_property_string($net_fmt, $data) };
1840     if ($@) {
1841         warn $@;
1842         return;
1843     }
1844     if (!defined($res->{macaddr})) {
1845         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1846         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1847     }
1848     return $res;
1849 }
1850
1851 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1852 sub parse_ipconfig {
1853     my ($data) = @_;
1854
1855     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1856     if ($@) {
1857         warn $@;
1858         return;
1859     }
1860
1861     if ($res->{gw} && !$res->{ip}) {
1862         warn 'gateway specified without specifying an IP address';
1863         return;
1864     }
1865     if ($res->{gw6} && !$res->{ip6}) {
1866         warn 'IPv6 gateway specified without specifying an IPv6 address';
1867         return;
1868     }
1869     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1870         warn 'gateway specified together with DHCP';
1871         return;
1872     }
1873     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1874         # gw6 + auto/dhcp
1875         warn "IPv6 gateway specified together with $res->{ip6} address";
1876         return;
1877     }
1878
1879     if (!$res->{ip} && !$res->{ip6}) {
1880         return { ip => 'dhcp', ip6 => 'dhcp' };
1881     }
1882
1883     return $res;
1884 }
1885
1886 sub print_net {
1887     my $net = shift;
1888
1889     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1890 }
1891
1892 sub add_random_macs {
1893     my ($settings) = @_;
1894
1895     foreach my $opt (keys %$settings) {
1896         next if $opt !~ m/^net(\d+)$/;
1897         my $net = parse_net($settings->{$opt});
1898         next if !$net;
1899         $settings->{$opt} = print_net($net);
1900     }
1901 }
1902
1903 sub vm_is_volid_owner {
1904     my ($storecfg, $vmid, $volid) = @_;
1905
1906     if ($volid !~  m|^/|) {
1907         my ($path, $owner);
1908         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1909         if ($owner && ($owner == $vmid)) {
1910             return 1;
1911         }
1912     }
1913
1914     return;
1915 }
1916
1917 sub vmconfig_register_unused_drive {
1918     my ($storecfg, $vmid, $conf, $drive) = @_;
1919
1920     if (drive_is_cloudinit($drive)) {
1921         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1922         warn $@ if $@;
1923     } elsif (!drive_is_cdrom($drive)) {
1924         my $volid = $drive->{file};
1925         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1926             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1927         }
1928     }
1929 }
1930
1931 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1932 my $smbios1_fmt = {
1933     uuid => {
1934         type => 'string',
1935         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1936         format_description => 'UUID',
1937         description => "Set SMBIOS1 UUID.",
1938         optional => 1,
1939     },
1940     version => {
1941         type => 'string',
1942         pattern => '[A-Za-z0-9+\/]+={0,2}',
1943         format_description => 'Base64 encoded string',
1944         description => "Set SMBIOS1 version.",
1945         optional => 1,
1946     },
1947     serial => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 serial number.",
1952         optional => 1,
1953     },
1954     manufacturer => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 manufacturer.",
1959         optional => 1,
1960     },
1961     product => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 product ID.",
1966         optional => 1,
1967     },
1968     sku => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 SKU string.",
1973         optional => 1,
1974     },
1975     family => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 family string.",
1980         optional => 1,
1981     },
1982     base64 => {
1983         type => 'boolean',
1984         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1985         optional => 1,
1986     },
1987 };
1988
1989 sub parse_smbios1 {
1990     my ($data) = @_;
1991
1992     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1993     warn $@ if $@;
1994     return $res;
1995 }
1996
1997 sub print_smbios1 {
1998     my ($smbios1) = @_;
1999     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2000 }
2001
2002 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2003
2004 sub parse_watchdog {
2005     my ($value) = @_;
2006
2007     return if !$value;
2008
2009     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2010     warn $@ if $@;
2011     return $res;
2012 }
2013
2014 sub parse_guest_agent {
2015     my ($conf) = @_;
2016
2017     return {} if !defined($conf->{agent});
2018
2019     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2020     warn $@ if $@;
2021
2022     # if the agent is disabled ignore the other potentially set properties
2023     return {} if !$res->{enabled};
2024     return $res;
2025 }
2026
2027 sub get_qga_key {
2028     my ($conf, $key) = @_;
2029     return undef if !defined($conf->{agent});
2030
2031     my $agent = parse_guest_agent($conf);
2032     return $agent->{$key};
2033 }
2034
2035 sub parse_vga {
2036     my ($value) = @_;
2037
2038     return {} if !$value;
2039     my $res = eval { parse_property_string($vga_fmt, $value) };
2040     warn $@ if $@;
2041     return $res;
2042 }
2043
2044 sub parse_rng {
2045     my ($value) = @_;
2046
2047     return if !$value;
2048
2049     my $res = eval { parse_property_string($rng_fmt, $value) };
2050     warn $@ if $@;
2051     return $res;
2052 }
2053
2054 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2055 sub verify_usb_device {
2056     my ($value, $noerr) = @_;
2057
2058     return $value if parse_usb_device($value);
2059
2060     return if $noerr;
2061
2062     die "unable to parse usb device\n";
2063 }
2064
2065 # add JSON properties for create and set function
2066 sub json_config_properties {
2067     my $prop = shift;
2068
2069     foreach my $opt (keys %$confdesc) {
2070         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2071             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2072         $prop->{$opt} = $confdesc->{$opt};
2073     }
2074
2075     return $prop;
2076 }
2077
2078 # return copy of $confdesc_cloudinit to generate documentation
2079 sub cloudinit_config_properties {
2080
2081     return dclone($confdesc_cloudinit);
2082 }
2083
2084 sub check_type {
2085     my ($key, $value) = @_;
2086
2087     die "unknown setting '$key'\n" if !$confdesc->{$key};
2088
2089     my $type = $confdesc->{$key}->{type};
2090
2091     if (!defined($value)) {
2092         die "got undefined value\n";
2093     }
2094
2095     if ($value =~ m/[\n\r]/) {
2096         die "property contains a line feed\n";
2097     }
2098
2099     if ($type eq 'boolean') {
2100         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2101         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2102         die "type check ('boolean') failed - got '$value'\n";
2103     } elsif ($type eq 'integer') {
2104         return int($1) if $value =~ m/^(\d+)$/;
2105         die "type check ('integer') failed - got '$value'\n";
2106     } elsif ($type eq 'number') {
2107         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2108         die "type check ('number') failed - got '$value'\n";
2109     } elsif ($type eq 'string') {
2110         if (my $fmt = $confdesc->{$key}->{format}) {
2111             PVE::JSONSchema::check_format($fmt, $value);
2112             return $value;
2113         }
2114         $value =~ s/^\"(.*)\"$/$1/;
2115         return $value;
2116     } else {
2117         die "internal error"
2118     }
2119 }
2120
2121 sub destroy_vm {
2122     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2123
2124     my $conf = PVE::QemuConfig->load_config($vmid);
2125
2126     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2127
2128     if ($conf->{template}) {
2129         # check if any base image is still used by a linked clone
2130         PVE::QemuConfig->foreach_volume($conf, sub {
2131                 my ($ds, $drive) = @_;
2132                 return if drive_is_cdrom($drive);
2133
2134                 my $volid = $drive->{file};
2135                 return if !$volid || $volid =~ m|^/|;
2136
2137                 die "base volume '$volid' is still in use by linked cloned\n"
2138                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2139
2140         });
2141     }
2142
2143     # only remove disks owned by this VM (referenced in the config)
2144     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2145         my ($ds, $drive) = @_;
2146         return if drive_is_cdrom($drive, 1);
2147
2148         my $volid = $drive->{file};
2149         return if !$volid || $volid =~ m|^/|;
2150
2151         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2152         return if !$path || !$owner || ($owner != $vmid);
2153
2154         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2155         warn "Could not remove disk '$volid', check manually: $@" if $@;
2156     });
2157
2158     if ($purge_unreferenced) { # also remove unreferenced disk
2159         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2160         PVE::Storage::foreach_volid($vmdisks, sub {
2161             my ($volid, $sid, $volname, $d) = @_;
2162             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2163             warn $@ if $@;
2164         });
2165     }
2166
2167     if (defined $replacement_conf) {
2168         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2169     } else {
2170         PVE::QemuConfig->destroy_config($vmid);
2171     }
2172 }
2173
2174 sub parse_vm_config {
2175     my ($filename, $raw) = @_;
2176
2177     return if !defined($raw);
2178
2179     my $res = {
2180         digest => Digest::SHA::sha1_hex($raw),
2181         snapshots => {},
2182         pending => {},
2183     };
2184
2185     $filename =~ m|/qemu-server/(\d+)\.conf$|
2186         || die "got strange filename '$filename'";
2187
2188     my $vmid = $1;
2189
2190     my $conf = $res;
2191     my $descr;
2192     my $section = '';
2193
2194     my @lines = split(/\n/, $raw);
2195     foreach my $line (@lines) {
2196         next if $line =~ m/^\s*$/;
2197
2198         if ($line =~ m/^\[PENDING\]\s*$/i) {
2199             $section = 'pending';
2200             if (defined($descr)) {
2201                 $descr =~ s/\s+$//;
2202                 $conf->{description} = $descr;
2203             }
2204             $descr = undef;
2205             $conf = $res->{$section} = {};
2206             next;
2207
2208         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2209             $section = $1;
2210             if (defined($descr)) {
2211                 $descr =~ s/\s+$//;
2212                 $conf->{description} = $descr;
2213             }
2214             $descr = undef;
2215             $conf = $res->{snapshots}->{$section} = {};
2216             next;
2217         }
2218
2219         if ($line =~ m/^\#(.*)\s*$/) {
2220             $descr = '' if !defined($descr);
2221             $descr .= PVE::Tools::decode_text($1) . "\n";
2222             next;
2223         }
2224
2225         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2226             $descr = '' if !defined($descr);
2227             $descr .= PVE::Tools::decode_text($2);
2228         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2229             $conf->{snapstate} = $1;
2230         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2231             my $key = $1;
2232             my $value = $2;
2233             $conf->{$key} = $value;
2234         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2235             my $value = $1;
2236             if ($section eq 'pending') {
2237                 $conf->{delete} = $value; # we parse this later
2238             } else {
2239                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2240             }
2241         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2242             my $key = $1;
2243             my $value = $2;
2244             eval { $value = check_type($key, $value); };
2245             if ($@) {
2246                 warn "vm $vmid - unable to parse value of '$key' - $@";
2247             } else {
2248                 $key = 'ide2' if $key eq 'cdrom';
2249                 my $fmt = $confdesc->{$key}->{format};
2250                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2251                     my $v = parse_drive($key, $value);
2252                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2253                         $v->{file} = $volid;
2254                         $value = print_drive($v);
2255                     } else {
2256                         warn "vm $vmid - unable to parse value of '$key'\n";
2257                         next;
2258                     }
2259                 }
2260
2261                 $conf->{$key} = $value;
2262             }
2263         } else {
2264             warn "vm $vmid - unable to parse config: $line\n";
2265         }
2266     }
2267
2268     if (defined($descr)) {
2269         $descr =~ s/\s+$//;
2270         $conf->{description} = $descr;
2271     }
2272     delete $res->{snapstate}; # just to be sure
2273
2274     return $res;
2275 }
2276
2277 sub write_vm_config {
2278     my ($filename, $conf) = @_;
2279
2280     delete $conf->{snapstate}; # just to be sure
2281
2282     if ($conf->{cdrom}) {
2283         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2284         $conf->{ide2} = $conf->{cdrom};
2285         delete $conf->{cdrom};
2286     }
2287
2288     # we do not use 'smp' any longer
2289     if ($conf->{sockets}) {
2290         delete $conf->{smp};
2291     } elsif ($conf->{smp}) {
2292         $conf->{sockets} = $conf->{smp};
2293         delete $conf->{cores};
2294         delete $conf->{smp};
2295     }
2296
2297     my $used_volids = {};
2298
2299     my $cleanup_config = sub {
2300         my ($cref, $pending, $snapname) = @_;
2301
2302         foreach my $key (keys %$cref) {
2303             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2304                 $key eq 'snapstate' || $key eq 'pending';
2305             my $value = $cref->{$key};
2306             if ($key eq 'delete') {
2307                 die "propertry 'delete' is only allowed in [PENDING]\n"
2308                     if !$pending;
2309                 # fixme: check syntax?
2310                 next;
2311             }
2312             eval { $value = check_type($key, $value); };
2313             die "unable to parse value of '$key' - $@" if $@;
2314
2315             $cref->{$key} = $value;
2316
2317             if (!$snapname && is_valid_drivename($key)) {
2318                 my $drive = parse_drive($key, $value);
2319                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2320             }
2321         }
2322     };
2323
2324     &$cleanup_config($conf);
2325
2326     &$cleanup_config($conf->{pending}, 1);
2327
2328     foreach my $snapname (keys %{$conf->{snapshots}}) {
2329         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2330         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2331     }
2332
2333     # remove 'unusedX' settings if we re-add a volume
2334     foreach my $key (keys %$conf) {
2335         my $value = $conf->{$key};
2336         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2337             delete $conf->{$key};
2338         }
2339     }
2340
2341     my $generate_raw_config = sub {
2342         my ($conf, $pending) = @_;
2343
2344         my $raw = '';
2345
2346         # add description as comment to top of file
2347         if (defined(my $descr = $conf->{description})) {
2348             if ($descr) {
2349                 foreach my $cl (split(/\n/, $descr)) {
2350                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2351                 }
2352             } else {
2353                 $raw .= "#\n" if $pending;
2354             }
2355         }
2356
2357         foreach my $key (sort keys %$conf) {
2358             next if $key =~ /^(digest|description|pending|snapshots)$/;
2359             $raw .= "$key: $conf->{$key}\n";
2360         }
2361         return $raw;
2362     };
2363
2364     my $raw = &$generate_raw_config($conf);
2365
2366     if (scalar(keys %{$conf->{pending}})){
2367         $raw .= "\n[PENDING]\n";
2368         $raw .= &$generate_raw_config($conf->{pending}, 1);
2369     }
2370
2371     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2372         $raw .= "\n[$snapname]\n";
2373         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2374     }
2375
2376     return $raw;
2377 }
2378
2379 sub load_defaults {
2380
2381     my $res = {};
2382
2383     # we use static defaults from our JSON schema configuration
2384     foreach my $key (keys %$confdesc) {
2385         if (defined(my $default = $confdesc->{$key}->{default})) {
2386             $res->{$key} = $default;
2387         }
2388     }
2389
2390     return $res;
2391 }
2392
2393 sub config_list {
2394     my $vmlist = PVE::Cluster::get_vmlist();
2395     my $res = {};
2396     return $res if !$vmlist || !$vmlist->{ids};
2397     my $ids = $vmlist->{ids};
2398     my $nodename = nodename();
2399
2400     foreach my $vmid (keys %$ids) {
2401         my $d = $ids->{$vmid};
2402         next if !$d->{node} || $d->{node} ne $nodename;
2403         next if !$d->{type} || $d->{type} ne 'qemu';
2404         $res->{$vmid}->{exists} = 1;
2405     }
2406     return $res;
2407 }
2408
2409 # test if VM uses local resources (to prevent migration)
2410 sub check_local_resources {
2411     my ($conf, $noerr) = @_;
2412
2413     my @loc_res = ();
2414
2415     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2416     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2417
2418     push @loc_res, "ivshmem" if $conf->{ivshmem};
2419
2420     foreach my $k (keys %$conf) {
2421         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2422         # sockets are safe: they will recreated be on the target side post-migrate
2423         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2424         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2425     }
2426
2427     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2428
2429     return \@loc_res;
2430 }
2431
2432 # check if used storages are available on all nodes (use by migrate)
2433 sub check_storage_availability {
2434     my ($storecfg, $conf, $node) = @_;
2435
2436     PVE::QemuConfig->foreach_volume($conf, sub {
2437         my ($ds, $drive) = @_;
2438
2439         my $volid = $drive->{file};
2440         return if !$volid;
2441
2442         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2443         return if !$sid;
2444
2445         # check if storage is available on both nodes
2446         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2447         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2448    });
2449 }
2450
2451 # list nodes where all VM images are available (used by has_feature API)
2452 sub shared_nodes {
2453     my ($conf, $storecfg) = @_;
2454
2455     my $nodelist = PVE::Cluster::get_nodelist();
2456     my $nodehash = { map { $_ => 1 } @$nodelist };
2457     my $nodename = nodename();
2458
2459     PVE::QemuConfig->foreach_volume($conf, sub {
2460         my ($ds, $drive) = @_;
2461
2462         my $volid = $drive->{file};
2463         return if !$volid;
2464
2465         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2466         if ($storeid) {
2467             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2468             if ($scfg->{disable}) {
2469                 $nodehash = {};
2470             } elsif (my $avail = $scfg->{nodes}) {
2471                 foreach my $node (keys %$nodehash) {
2472                     delete $nodehash->{$node} if !$avail->{$node};
2473                 }
2474             } elsif (!$scfg->{shared}) {
2475                 foreach my $node (keys %$nodehash) {
2476                     delete $nodehash->{$node} if $node ne $nodename
2477                 }
2478             }
2479         }
2480     });
2481
2482     return $nodehash
2483 }
2484
2485 sub check_local_storage_availability {
2486     my ($conf, $storecfg) = @_;
2487
2488     my $nodelist = PVE::Cluster::get_nodelist();
2489     my $nodehash = { map { $_ => {} } @$nodelist };
2490
2491     PVE::QemuConfig->foreach_volume($conf, sub {
2492         my ($ds, $drive) = @_;
2493
2494         my $volid = $drive->{file};
2495         return if !$volid;
2496
2497         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2498         if ($storeid) {
2499             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2500
2501             if ($scfg->{disable}) {
2502                 foreach my $node (keys %$nodehash) {
2503                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2504                 }
2505             } elsif (my $avail = $scfg->{nodes}) {
2506                 foreach my $node (keys %$nodehash) {
2507                     if (!$avail->{$node}) {
2508                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2509                     }
2510                 }
2511             }
2512         }
2513     });
2514
2515     foreach my $node (values %$nodehash) {
2516         if (my $unavail = $node->{unavailable_storages}) {
2517             $node->{unavailable_storages} = [ sort keys %$unavail ];
2518         }
2519     }
2520
2521     return $nodehash
2522 }
2523
2524 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2525 sub check_running {
2526     my ($vmid, $nocheck, $node) = @_;
2527
2528     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2529     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2530 }
2531
2532 sub vzlist {
2533
2534     my $vzlist = config_list();
2535
2536     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2537
2538     while (defined(my $de = $fd->read)) {
2539         next if $de !~ m/^(\d+)\.pid$/;
2540         my $vmid = $1;
2541         next if !defined($vzlist->{$vmid});
2542         if (my $pid = check_running($vmid)) {
2543             $vzlist->{$vmid}->{pid} = $pid;
2544         }
2545     }
2546
2547     return $vzlist;
2548 }
2549
2550 our $vmstatus_return_properties = {
2551     vmid => get_standard_option('pve-vmid'),
2552     status => {
2553         description => "Qemu process status.",
2554         type => 'string',
2555         enum => ['stopped', 'running'],
2556     },
2557     maxmem => {
2558         description => "Maximum memory in bytes.",
2559         type => 'integer',
2560         optional => 1,
2561         renderer => 'bytes',
2562     },
2563     maxdisk => {
2564         description => "Root disk size in bytes.",
2565         type => 'integer',
2566         optional => 1,
2567         renderer => 'bytes',
2568     },
2569     name => {
2570         description => "VM name.",
2571         type => 'string',
2572         optional => 1,
2573     },
2574     qmpstatus => {
2575         description => "Qemu QMP agent status.",
2576         type => 'string',
2577         optional => 1,
2578     },
2579     pid => {
2580         description => "PID of running qemu process.",
2581         type => 'integer',
2582         optional => 1,
2583     },
2584     uptime => {
2585         description => "Uptime.",
2586         type => 'integer',
2587         optional => 1,
2588         renderer => 'duration',
2589     },
2590     cpus => {
2591         description => "Maximum usable CPUs.",
2592         type => 'number',
2593         optional => 1,
2594     },
2595     lock => {
2596         description => "The current config lock, if any.",
2597         type => 'string',
2598         optional => 1,
2599     },
2600     tags => {
2601         description  => "The current configured tags, if any",
2602         type => 'string',
2603         optional => 1,
2604     },
2605     'running-machine' => {
2606         description => "The currently running machine type (if running).",
2607         type => 'string',
2608         optional => 1,
2609     },
2610     'running-qemu' => {
2611         description => "The currently running QEMU version (if running).",
2612         type => 'string',
2613         optional => 1,
2614     },
2615 };
2616
2617 my $last_proc_pid_stat;
2618
2619 # get VM status information
2620 # This must be fast and should not block ($full == false)
2621 # We only query KVM using QMP if $full == true (this can be slow)
2622 sub vmstatus {
2623     my ($opt_vmid, $full) = @_;
2624
2625     my $res = {};
2626
2627     my $storecfg = PVE::Storage::config();
2628
2629     my $list = vzlist();
2630     my $defaults = load_defaults();
2631
2632     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2633
2634     my $cpucount = $cpuinfo->{cpus} || 1;
2635
2636     foreach my $vmid (keys %$list) {
2637         next if $opt_vmid && ($vmid ne $opt_vmid);
2638
2639         my $conf = PVE::QemuConfig->load_config($vmid);
2640
2641         my $d = { vmid => $vmid };
2642         $d->{pid} = $list->{$vmid}->{pid};
2643
2644         # fixme: better status?
2645         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2646
2647         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2648         if (defined($size)) {
2649             $d->{disk} = 0; # no info available
2650             $d->{maxdisk} = $size;
2651         } else {
2652             $d->{disk} = 0;
2653             $d->{maxdisk} = 0;
2654         }
2655
2656         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2657             * ($conf->{cores} || $defaults->{cores});
2658         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2659         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2660
2661         $d->{name} = $conf->{name} || "VM $vmid";
2662         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2663             : $defaults->{memory}*(1024*1024);
2664
2665         if ($conf->{balloon}) {
2666             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2667             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2668                 : $defaults->{shares};
2669         }
2670
2671         $d->{uptime} = 0;
2672         $d->{cpu} = 0;
2673         $d->{mem} = 0;
2674
2675         $d->{netout} = 0;
2676         $d->{netin} = 0;
2677
2678         $d->{diskread} = 0;
2679         $d->{diskwrite} = 0;
2680
2681         $d->{template} = PVE::QemuConfig->is_template($conf);
2682
2683         $d->{serial} = 1 if conf_has_serial($conf);
2684         $d->{lock} = $conf->{lock} if $conf->{lock};
2685         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2686
2687         $res->{$vmid} = $d;
2688     }
2689
2690     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2691     foreach my $dev (keys %$netdev) {
2692         next if $dev !~ m/^tap([1-9]\d*)i/;
2693         my $vmid = $1;
2694         my $d = $res->{$vmid};
2695         next if !$d;
2696
2697         $d->{netout} += $netdev->{$dev}->{receive};
2698         $d->{netin} += $netdev->{$dev}->{transmit};
2699
2700         if ($full) {
2701             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2702             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2703         }
2704
2705     }
2706
2707     my $ctime = gettimeofday;
2708
2709     foreach my $vmid (keys %$list) {
2710
2711         my $d = $res->{$vmid};
2712         my $pid = $d->{pid};
2713         next if !$pid;
2714
2715         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2716         next if !$pstat; # not running
2717
2718         my $used = $pstat->{utime} + $pstat->{stime};
2719
2720         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2721
2722         if ($pstat->{vsize}) {
2723             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2724         }
2725
2726         my $old = $last_proc_pid_stat->{$pid};
2727         if (!$old) {
2728             $last_proc_pid_stat->{$pid} = {
2729                 time => $ctime,
2730                 used => $used,
2731                 cpu => 0,
2732             };
2733             next;
2734         }
2735
2736         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2737
2738         if ($dtime > 1000) {
2739             my $dutime = $used -  $old->{used};
2740
2741             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2742             $last_proc_pid_stat->{$pid} = {
2743                 time => $ctime,
2744                 used => $used,
2745                 cpu => $d->{cpu},
2746             };
2747         } else {
2748             $d->{cpu} = $old->{cpu};
2749         }
2750     }
2751
2752     return $res if !$full;
2753
2754     my $qmpclient = PVE::QMPClient->new();
2755
2756     my $ballooncb = sub {
2757         my ($vmid, $resp) = @_;
2758
2759         my $info = $resp->{'return'};
2760         return if !$info->{max_mem};
2761
2762         my $d = $res->{$vmid};
2763
2764         # use memory assigned to VM
2765         $d->{maxmem} = $info->{max_mem};
2766         $d->{balloon} = $info->{actual};
2767
2768         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2769             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2770             $d->{freemem} = $info->{free_mem};
2771         }
2772
2773         $d->{ballooninfo} = $info;
2774     };
2775
2776     my $blockstatscb = sub {
2777         my ($vmid, $resp) = @_;
2778         my $data = $resp->{'return'} || [];
2779         my $totalrdbytes = 0;
2780         my $totalwrbytes = 0;
2781
2782         for my $blockstat (@$data) {
2783             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2784             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2785
2786             $blockstat->{device} =~ s/drive-//;
2787             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2788         }
2789         $res->{$vmid}->{diskread} = $totalrdbytes;
2790         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2791     };
2792
2793     my $machinecb = sub {
2794         my ($vmid, $resp) = @_;
2795         my $data = $resp->{'return'} || [];
2796
2797         $res->{$vmid}->{'running-machine'} =
2798             PVE::QemuServer::Machine::current_from_query_machines($data);
2799     };
2800
2801     my $versioncb = sub {
2802         my ($vmid, $resp) = @_;
2803         my $data = $resp->{'return'} // {};
2804         my $version = 'unknown';
2805
2806         if (my $v = $data->{qemu}) {
2807             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2808         }
2809
2810         $res->{$vmid}->{'running-qemu'} = $version;
2811     };
2812
2813     my $statuscb = sub {
2814         my ($vmid, $resp) = @_;
2815
2816         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2817         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2818         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2819         # this fails if ballon driver is not loaded, so this must be
2820         # the last commnand (following command are aborted if this fails).
2821         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2822
2823         my $status = 'unknown';
2824         if (!defined($status = $resp->{'return'}->{status})) {
2825             warn "unable to get VM status\n";
2826             return;
2827         }
2828
2829         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2830     };
2831
2832     foreach my $vmid (keys %$list) {
2833         next if $opt_vmid && ($vmid ne $opt_vmid);
2834         next if !$res->{$vmid}->{pid}; # not running
2835         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2836     }
2837
2838     $qmpclient->queue_execute(undef, 2);
2839
2840     foreach my $vmid (keys %$list) {
2841         next if $opt_vmid && ($vmid ne $opt_vmid);
2842         next if !$res->{$vmid}->{pid}; #not running
2843
2844         # we can't use the $qmpclient since it might have already aborted on
2845         # 'query-balloon', but this might also fail for older versions...
2846         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2847         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2848     }
2849
2850     foreach my $vmid (keys %$list) {
2851         next if $opt_vmid && ($vmid ne $opt_vmid);
2852         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2853     }
2854
2855     return $res;
2856 }
2857
2858 sub conf_has_serial {
2859     my ($conf) = @_;
2860
2861     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2862         if ($conf->{"serial$i"}) {
2863             return 1;
2864         }
2865     }
2866
2867     return 0;
2868 }
2869
2870 sub conf_has_audio {
2871     my ($conf, $id) = @_;
2872
2873     $id //= 0;
2874     my $audio = $conf->{"audio$id"};
2875     return if !defined($audio);
2876
2877     my $audioproperties = parse_property_string($audio_fmt, $audio);
2878     my $audiodriver = $audioproperties->{driver} // 'spice';
2879
2880     return {
2881         dev => $audioproperties->{device},
2882         dev_id => "audiodev$id",
2883         backend => $audiodriver,
2884         backend_id => "$audiodriver-backend${id}",
2885     };
2886 }
2887
2888 sub audio_devs {
2889     my ($audio, $audiopciaddr, $machine_version) = @_;
2890
2891     my $devs = [];
2892
2893     my $id = $audio->{dev_id};
2894     my $audiodev = "";
2895     if (min_version($machine_version, 4, 2)) {
2896         $audiodev = ",audiodev=$audio->{backend_id}";
2897     }
2898
2899     if ($audio->{dev} eq 'AC97') {
2900         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2901     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2902         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2903         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2904         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2905     } else {
2906         die "unkown audio device '$audio->{dev}', implement me!";
2907     }
2908
2909     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2910
2911     return $devs;
2912 }
2913
2914 sub vga_conf_has_spice {
2915     my ($vga) = @_;
2916
2917     my $vgaconf = parse_vga($vga);
2918     my $vgatype = $vgaconf->{type};
2919     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2920
2921     return $1 || 1;
2922 }
2923
2924 sub is_native($) {
2925     my ($arch) = @_;
2926     return get_host_arch() eq $arch;
2927 }
2928
2929 sub get_vm_arch {
2930     my ($conf) = @_;
2931     return $conf->{arch} // get_host_arch();
2932 }
2933
2934 my $default_machines = {
2935     x86_64 => 'pc',
2936     aarch64 => 'virt',
2937 };
2938
2939 sub get_installed_machine_version {
2940     my ($kvmversion) = @_;
2941     $kvmversion = kvm_user_version() if !defined($kvmversion);
2942     $kvmversion =~ m/^(\d+\.\d+)/;
2943     return $1;
2944 }
2945
2946 sub windows_get_pinned_machine_version {
2947     my ($machine, $base_version, $kvmversion) = @_;
2948
2949     my $pin_version = $base_version;
2950     if (!defined($base_version) ||
2951         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2952     ) {
2953         $pin_version = get_installed_machine_version($kvmversion);
2954     }
2955     if (!$machine || $machine eq 'pc') {
2956         $machine = "pc-i440fx-$pin_version";
2957     } elsif ($machine eq 'q35') {
2958         $machine = "pc-q35-$pin_version";
2959     } elsif ($machine eq 'virt') {
2960         $machine = "virt-$pin_version";
2961     } else {
2962         warn "unknown machine type '$machine', not touching that!\n";
2963     }
2964
2965     return $machine;
2966 }
2967
2968 sub get_vm_machine {
2969     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2970
2971     my $machine = $forcemachine || $conf->{machine};
2972
2973     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2974         $kvmversion //= kvm_user_version();
2975         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2976         # layout which confuses windows quite a bit and may result in various regressions..
2977         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2978         if (windows_version($conf->{ostype})) {
2979             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2980         }
2981         $arch //= 'x86_64';
2982         $machine ||= $default_machines->{$arch};
2983         if ($add_pve_version) {
2984             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2985             $machine .= "+pve$pvever";
2986         }
2987     }
2988
2989     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
2990         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
2991         $machine = $1 if $is_pxe;
2992
2993         # for version-pinned machines that do not include a pve-version (e.g.
2994         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2995         $machine .= '+pve0';
2996
2997         $machine .= '.pxe' if $is_pxe;
2998     }
2999
3000     return $machine;
3001 }
3002
3003 sub get_ovmf_files($) {
3004     my ($arch) = @_;
3005
3006     my $ovmf = $OVMF->{$arch}
3007         or die "no OVMF images known for architecture '$arch'\n";
3008
3009     return @$ovmf;
3010 }
3011
3012 my $Arch2Qemu = {
3013     aarch64 => '/usr/bin/qemu-system-aarch64',
3014     x86_64 => '/usr/bin/qemu-system-x86_64',
3015 };
3016 sub get_command_for_arch($) {
3017     my ($arch) = @_;
3018     return '/usr/bin/kvm' if is_native($arch);
3019
3020     my $cmd = $Arch2Qemu->{$arch}
3021         or die "don't know how to emulate architecture '$arch'\n";
3022     return $cmd;
3023 }
3024
3025 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3026 # to use in a QEMU command line (-cpu element), first array_intersect the result
3027 # of query_supported_ with query_understood_. This is necessary because:
3028 #
3029 # a) query_understood_ returns flags the host cannot use and
3030 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3031 #    flags, but CPU settings - with most of them being flags. Those settings
3032 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3033 #
3034 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3035 # expensive. If you need the value returned from this, you can get it much
3036 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3037 # $accel being 'kvm' or 'tcg'.
3038 #
3039 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3040 # changes, automatically populating pmxcfs.
3041 #
3042 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3043 # since kvm and tcg machines support different flags
3044 #
3045 sub query_supported_cpu_flags {
3046     my ($arch) = @_;
3047
3048     $arch //= get_host_arch();
3049     my $default_machine = $default_machines->{$arch};
3050
3051     my $flags = {};
3052
3053     # FIXME: Once this is merged, the code below should work for ARM as well:
3054     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3055     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3056         $arch eq "aarch64";
3057
3058     my $kvm_supported = defined(kvm_version());
3059     my $qemu_cmd = get_command_for_arch($arch);
3060     my $fakevmid = -1;
3061     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3062
3063     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3064     my $query_supported_run_qemu = sub {
3065         my ($kvm) = @_;
3066
3067         my $flags = {};
3068         my $cmd = [
3069             $qemu_cmd,
3070             '-machine', $default_machine,
3071             '-display', 'none',
3072             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3073             '-mon', 'chardev=qmp,mode=control',
3074             '-pidfile', $pidfile,
3075             '-S', '-daemonize'
3076         ];
3077
3078         if (!$kvm) {
3079             push @$cmd, '-accel', 'tcg';
3080         }
3081
3082         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3083         die "QEMU flag querying VM exited with code " . $rc if $rc;
3084
3085         eval {
3086             my $cmd_result = mon_cmd(
3087                 $fakevmid,
3088                 'query-cpu-model-expansion',
3089                 type => 'full',
3090                 model => { name => 'host' }
3091             );
3092
3093             my $props = $cmd_result->{model}->{props};
3094             foreach my $prop (keys %$props) {
3095                 next if $props->{$prop} ne '1';
3096                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3097                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3098                 # We only keep those with underscores, to match /proc/cpuinfo
3099                 $prop =~ s/\.|-/_/g;
3100                 $flags->{$prop} = 1;
3101             }
3102         };
3103         my $err = $@;
3104
3105         # force stop with 10 sec timeout and 'nocheck'
3106         # always stop, even if QMP failed
3107         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3108
3109         die $err if $err;
3110
3111         return [ sort keys %$flags ];
3112     };
3113
3114     # We need to query QEMU twice, since KVM and TCG have different supported flags
3115     PVE::QemuConfig->lock_config($fakevmid, sub {
3116         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3117         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3118
3119         if ($kvm_supported) {
3120             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3121             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3122         }
3123     });
3124
3125     return $flags;
3126 }
3127
3128 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3129 my $understood_cpu_flag_dir = "/usr/share/kvm";
3130 sub query_understood_cpu_flags {
3131     my $arch = get_host_arch();
3132     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3133
3134     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3135         if ! -e $filepath;
3136
3137     my $raw = file_get_contents($filepath);
3138     $raw =~ s/^\s+|\s+$//g;
3139     my @flags = split(/\s+/, $raw);
3140
3141     return \@flags;
3142 }
3143
3144 sub config_to_command {
3145     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3146         $pbs_backing) = @_;
3147
3148     my $cmd = [];
3149     my $globalFlags = [];
3150     my $machineFlags = [];
3151     my $rtcFlags = [];
3152     my $devices = [];
3153     my $pciaddr = '';
3154     my $bridges = {};
3155     my $ostype = $conf->{ostype};
3156     my $winversion = windows_version($ostype);
3157     my $kvm = $conf->{kvm};
3158     my $nodename = nodename();
3159
3160     my $arch = get_vm_arch($conf);
3161     my $kvm_binary = get_command_for_arch($arch);
3162     my $kvmver = kvm_user_version($kvm_binary);
3163
3164     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3165         $kvmver //= "undefined";
3166         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3167     }
3168
3169     my $add_pve_version = min_version($kvmver, 4, 1);
3170
3171     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3172     my $machine_version = extract_version($machine_type, $kvmver);
3173     $kvm //= 1 if is_native($arch);
3174
3175     $machine_version =~ m/(\d+)\.(\d+)/;
3176     my ($machine_major, $machine_minor) = ($1, $2);
3177
3178     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3179         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3180     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3181         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3182             ." please upgrade node '$nodename'\n"
3183     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3184         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3185         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3186             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3187             ." node '$nodename'\n";
3188     }
3189
3190     # if a specific +pve version is required for a feature, use $version_guard
3191     # instead of min_version to allow machines to be run with the minimum
3192     # required version
3193     my $required_pve_version = 0;
3194     my $version_guard = sub {
3195         my ($major, $minor, $pve) = @_;
3196         return 0 if !min_version($machine_version, $major, $minor, $pve);
3197         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3198         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3199         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3200         return 1;
3201     };
3202
3203     if ($kvm && !defined kvm_version()) {
3204         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3205             ." or enable in BIOS.\n";
3206     }
3207
3208     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3209     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3210     my $use_old_bios_files = undef;
3211     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3212
3213     my $cpuunits = defined($conf->{cpuunits}) ?
3214             $conf->{cpuunits} : $defaults->{cpuunits};
3215
3216     push @$cmd, $kvm_binary;
3217
3218     push @$cmd, '-id', $vmid;
3219
3220     my $vmname = $conf->{name} || "vm$vmid";
3221
3222     push @$cmd, '-name', $vmname;
3223
3224     push @$cmd, '-no-shutdown';
3225
3226     my $use_virtio = 0;
3227
3228     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3229     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3230     push @$cmd, '-mon', "chardev=qmp,mode=control";
3231
3232     if (min_version($machine_version, 2, 12)) {
3233         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3234         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3235     }
3236
3237     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3238
3239     push @$cmd, '-daemonize';
3240
3241     if ($conf->{smbios1}) {
3242         my $smbios_conf = parse_smbios1($conf->{smbios1});
3243         if ($smbios_conf->{base64}) {
3244             # Do not pass base64 flag to qemu
3245             delete $smbios_conf->{base64};
3246             my $smbios_string = "";
3247             foreach my $key (keys %$smbios_conf) {
3248                 my $value;
3249                 if ($key eq "uuid") {
3250                     $value = $smbios_conf->{uuid}
3251                 } else {
3252                     $value = decode_base64($smbios_conf->{$key});
3253                 }
3254                 # qemu accepts any binary data, only commas need escaping by double comma
3255                 $value =~ s/,/,,/g;
3256                 $smbios_string .= "," . $key . "=" . $value if $value;
3257             }
3258             push @$cmd, '-smbios', "type=1" . $smbios_string;
3259         } else {
3260             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3261         }
3262     }
3263
3264     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3265         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3266         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3267
3268         my ($path, $format);
3269         if (my $efidisk = $conf->{efidisk0}) {
3270             my $d = parse_drive('efidisk0', $efidisk);
3271             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3272             $format = $d->{format};
3273             if ($storeid) {
3274                 $path = PVE::Storage::path($storecfg, $d->{file});
3275                 if (!defined($format)) {
3276                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3277                     $format = qemu_img_format($scfg, $volname);
3278                 }
3279             } else {
3280                 $path = $d->{file};
3281                 die "efidisk format must be specified\n"
3282                     if !defined($format);
3283             }
3284         } else {
3285             warn "no efidisk configured! Using temporary efivars disk.\n";
3286             $path = "/tmp/$vmid-ovmf.fd";
3287             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3288             $format = 'raw';
3289         }
3290
3291         my $size_str = "";
3292
3293         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3294             $size_str = ",size=" . (-s $ovmf_vars);
3295         }
3296
3297         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3298         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3299     }
3300
3301     # load q35 config
3302     if ($q35) {
3303         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3304         if (min_version($machine_version, 4, 0)) {
3305             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3306         } else {
3307             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3308         }
3309     }
3310
3311     if ($conf->{vmgenid}) {
3312         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3313     }
3314
3315     # add usb controllers
3316     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3317         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3318     push @$devices, @usbcontrollers if @usbcontrollers;
3319     my $vga = parse_vga($conf->{vga});
3320
3321     my $qxlnum = vga_conf_has_spice($conf->{vga});
3322     $vga->{type} = 'qxl' if $qxlnum;
3323
3324     if (!$vga->{type}) {
3325         if ($arch eq 'aarch64') {
3326             $vga->{type} = 'virtio';
3327         } elsif (min_version($machine_version, 2, 9)) {
3328             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3329         } else {
3330             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3331         }
3332     }
3333
3334     # enable absolute mouse coordinates (needed by vnc)
3335     my $tablet;
3336     if (defined($conf->{tablet})) {
3337         $tablet = $conf->{tablet};
3338     } else {
3339         $tablet = $defaults->{tablet};
3340         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3341         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3342     }
3343
3344     if ($tablet) {
3345         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3346         my $kbd = print_keyboarddevice_full($conf, $arch);
3347         push @$devices, '-device', $kbd if defined($kbd);
3348     }
3349
3350     my $bootorder = device_bootorder($conf);
3351
3352     # host pci device passthrough
3353     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3354         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3355
3356     # usb devices
3357     my $usb_dev_features = {};
3358     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3359
3360     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3361         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3362     push @$devices, @usbdevices if @usbdevices;
3363
3364     # serial devices
3365     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3366         if (my $path = $conf->{"serial$i"}) {
3367             if ($path eq 'socket') {
3368                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3369                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3370                 # On aarch64, serial0 is the UART device. Qemu only allows
3371                 # connecting UART devices via the '-serial' command line, as
3372                 # the device has a fixed slot on the hardware...
3373                 if ($arch eq 'aarch64' && $i == 0) {
3374                     push @$devices, '-serial', "chardev:serial$i";
3375                 } else {
3376                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3377                 }
3378             } else {
3379                 die "no such serial device\n" if ! -c $path;
3380                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3381                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3382             }
3383         }
3384     }
3385
3386     # parallel devices
3387     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3388         if (my $path = $conf->{"parallel$i"}) {
3389             die "no such parallel device\n" if ! -c $path;
3390             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3391             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3392             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3393         }
3394     }
3395
3396     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3397         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3398         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3399         push @$devices, @$audio_devs;
3400     }
3401
3402     my $sockets = 1;
3403     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3404     $sockets = $conf->{sockets} if  $conf->{sockets};
3405
3406     my $cores = $conf->{cores} || 1;
3407
3408     my $maxcpus = $sockets * $cores;
3409
3410     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3411
3412     my $allowed_vcpus = $cpuinfo->{cpus};
3413
3414     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3415         if ($allowed_vcpus < $maxcpus);
3416
3417     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3418
3419         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3420         for (my $i = 2; $i <= $vcpus; $i++)  {
3421             my $cpustr = print_cpu_device($conf,$i);
3422             push @$cmd, '-device', $cpustr;
3423         }
3424
3425     } else {
3426
3427         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3428     }
3429     push @$cmd, '-nodefaults';
3430
3431     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3432
3433     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3434
3435     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3436
3437     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3438         push @$devices, '-device', print_vga_device(
3439             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3440         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3441         push @$cmd,  '-vnc', "unix:$socket,password";
3442     } else {
3443         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3444         push @$cmd, '-nographic';
3445     }
3446
3447     # time drift fix
3448     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3449     my $useLocaltime = $conf->{localtime};
3450
3451     if ($winversion >= 5) { # windows
3452         $useLocaltime = 1 if !defined($conf->{localtime});
3453
3454         # use time drift fix when acpi is enabled
3455         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3456             $tdf = 1 if !defined($conf->{tdf});
3457         }
3458     }
3459
3460     if ($winversion >= 6) {
3461         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3462         push @$cmd, '-no-hpet';
3463     }
3464
3465     push @$rtcFlags, 'driftfix=slew' if $tdf;
3466
3467     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3468         push @$rtcFlags, "base=$conf->{startdate}";
3469     } elsif ($useLocaltime) {
3470         push @$rtcFlags, 'base=localtime';
3471     }
3472
3473     if ($forcecpu) {
3474         push @$cmd, '-cpu', $forcecpu;
3475     } else {
3476         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3477     }
3478
3479     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3480
3481     push @$cmd, '-S' if $conf->{freeze};
3482
3483     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3484
3485     my $guest_agent = parse_guest_agent($conf);
3486
3487     if ($guest_agent->{enabled}) {
3488         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3489         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3490
3491         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3492             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3493             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3494             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3495         } elsif ($guest_agent->{type} eq 'isa') {
3496             push @$devices, '-device', "isa-serial,chardev=qga0";
3497         }
3498     }
3499
3500     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3501     if ($rng && $version_guard->(4, 1, 2)) {
3502         check_rng_source($rng->{source});
3503
3504         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3505         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3506         my $limiter_str = "";
3507         if ($max_bytes) {
3508             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3509         }
3510
3511         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3512         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3513         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3514     }
3515
3516     my $spice_port;
3517
3518     if ($qxlnum) {
3519         if ($qxlnum > 1) {
3520             if ($winversion){
3521                 for (my $i = 1; $i < $qxlnum; $i++){
3522                     push @$devices, '-device', print_vga_device(
3523                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3524                 }
3525             } else {
3526                 # assume other OS works like Linux
3527                 my ($ram, $vram) = ("134217728", "67108864");
3528                 if ($vga->{memory}) {
3529                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3530                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3531                 }
3532                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3533                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3534             }
3535         }
3536
3537         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3538
3539         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3540         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3541         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3542
3543         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3544         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3545         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3546
3547         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3548         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3549
3550         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3551         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3552         if ($spice_enhancement->{foldersharing}) {
3553             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3554             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3555         }
3556
3557         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3558         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3559             if $spice_enhancement->{videostreaming};
3560
3561         push @$devices, '-spice', "$spice_opts";
3562     }
3563
3564     # enable balloon by default, unless explicitly disabled
3565     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3566         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3567         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3568     }
3569
3570     if ($conf->{watchdog}) {
3571         my $wdopts = parse_watchdog($conf->{watchdog});
3572         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3573         my $watchdog = $wdopts->{model} || 'i6300esb';
3574         push @$devices, '-device', "$watchdog$pciaddr";
3575         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3576     }
3577
3578     my $vollist = [];
3579     my $scsicontroller = {};
3580     my $ahcicontroller = {};
3581     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3582
3583     # Add iscsi initiator name if available
3584     if (my $initiator = get_initiator_name()) {
3585         push @$devices, '-iscsi', "initiator-name=$initiator";
3586     }
3587
3588     PVE::QemuConfig->foreach_volume($conf, sub {
3589         my ($ds, $drive) = @_;
3590
3591         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3592             push @$vollist, $drive->{file};
3593         }
3594
3595         # ignore efidisk here, already added in bios/fw handling code above
3596         return if $drive->{interface} eq 'efidisk';
3597
3598         $use_virtio = 1 if $ds =~ m/^virtio/;
3599
3600         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3601
3602         if ($drive->{interface} eq 'virtio'){
3603            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3604         }
3605
3606         if ($drive->{interface} eq 'scsi') {
3607
3608             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3609
3610             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3611                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3612
3613             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3614             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3615
3616             my $iothread = '';
3617             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3618                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3619                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3620             } elsif ($drive->{iothread}) {
3621                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3622             }
3623
3624             my $queues = '';
3625             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3626                 $queues = ",num_queues=$drive->{queues}";
3627             }
3628
3629             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3630                 if !$scsicontroller->{$controller};
3631             $scsicontroller->{$controller}=1;
3632         }
3633
3634         if ($drive->{interface} eq 'sata') {
3635             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3636             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3637             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3638                 if !$ahcicontroller->{$controller};
3639             $ahcicontroller->{$controller}=1;
3640         }
3641
3642         my $pbs_conf = $pbs_backing->{$ds};
3643         my $pbs_name = undef;
3644         if ($pbs_conf) {
3645             $pbs_name = "drive-$ds-pbs";
3646             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3647         }
3648
3649         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3650         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3651
3652         push @$devices, '-drive',$drive_cmd;
3653         push @$devices, '-device', print_drivedevice_full(
3654             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3655     });
3656
3657     for (my $i = 0; $i < $MAX_NETS; $i++) {
3658         my $netname = "net$i";
3659
3660         next if !$conf->{$netname};
3661         my $d = parse_net($conf->{$netname});
3662         next if !$d;
3663
3664         $use_virtio = 1 if $d->{model} eq 'virtio';
3665
3666         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3667
3668         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3669         push @$devices, '-netdev', $netdevfull;
3670
3671         my $netdevicefull = print_netdevice_full(
3672             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3673
3674         push @$devices, '-device', $netdevicefull;
3675     }
3676
3677     if ($conf->{ivshmem}) {
3678         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3679
3680         my $bus;
3681         if ($q35) {
3682             $bus = print_pcie_addr("ivshmem");
3683         } else {
3684             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3685         }
3686
3687         my $ivshmem_name = $ivshmem->{name} // $vmid;
3688         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3689
3690         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3691         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3692             .",size=$ivshmem->{size}M";
3693     }
3694
3695     # pci.4 is nested in pci.1
3696     $bridges->{1} = 1 if $bridges->{4};
3697
3698     if (!$q35) {
3699         # add pci bridges
3700         if (min_version($machine_version, 2, 3)) {
3701            $bridges->{1} = 1;
3702            $bridges->{2} = 1;
3703         }
3704
3705         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3706
3707     }
3708
3709     for my $k (sort {$b cmp $a} keys %$bridges) {
3710         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3711
3712         my $k_name = $k;
3713         if ($k == 2 && $legacy_igd) {
3714             $k_name = "$k-igd";
3715         }
3716         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3717
3718         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3719         if ($q35) {
3720             # add after -readconfig pve-q35.cfg
3721             splice @$devices, 2, 0, '-device', $devstr;
3722         } else {
3723             unshift @$devices, '-device', $devstr if $k > 0;
3724         }
3725     }
3726
3727     if (!$kvm) {
3728         push @$machineFlags, 'accel=tcg';
3729     }
3730
3731     my $machine_type_min = $machine_type;
3732     if ($add_pve_version) {
3733         $machine_type_min =~ s/\+pve\d+$//;
3734         $machine_type_min .= "+pve$required_pve_version";
3735     }
3736     push @$machineFlags, "type=${machine_type_min}";
3737
3738     push @$cmd, @$devices;
3739     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3740     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3741     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3742
3743     if (my $vmstate = $conf->{vmstate}) {
3744         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3745         push @$vollist, $vmstate;
3746         push @$cmd, '-loadstate', $statepath;
3747         print "activating and using '$vmstate' as vmstate\n";
3748     }
3749
3750     # add custom args
3751     if ($conf->{args}) {
3752         my $aa = PVE::Tools::split_args($conf->{args});
3753         push @$cmd, @$aa;
3754     }
3755
3756     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3757 }
3758
3759 sub check_rng_source {
3760     my ($source) = @_;
3761
3762     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3763     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3764         if ! -e $source;
3765
3766     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3767     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3768         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3769         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3770         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3771             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3772             ." to the host.\n";
3773     }
3774 }
3775
3776 sub spice_port {
3777     my ($vmid) = @_;
3778
3779     my $res = mon_cmd($vmid, 'query-spice');
3780
3781     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3782 }
3783
3784 sub vm_devices_list {
3785     my ($vmid) = @_;
3786
3787     my $res = mon_cmd($vmid, 'query-pci');
3788     my $devices_to_check = [];
3789     my $devices = {};
3790     foreach my $pcibus (@$res) {
3791         push @$devices_to_check, @{$pcibus->{devices}},
3792     }
3793
3794     while (@$devices_to_check) {
3795         my $to_check = [];
3796         for my $d (@$devices_to_check) {
3797             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3798             next if !$d->{'pci_bridge'};
3799
3800             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3801             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3802         }
3803         $devices_to_check = $to_check;
3804     }
3805
3806     my $resblock = mon_cmd($vmid, 'query-block');
3807     foreach my $block (@$resblock) {
3808         if($block->{device} =~ m/^drive-(\S+)/){
3809                 $devices->{$1} = 1;
3810         }
3811     }
3812
3813     my $resmice = mon_cmd($vmid, 'query-mice');
3814     foreach my $mice (@$resmice) {
3815         if ($mice->{name} eq 'QEMU HID Tablet') {
3816             $devices->{tablet} = 1;
3817             last;
3818         }
3819     }
3820
3821     # for usb devices there is no query-usb
3822     # but we can iterate over the entries in
3823     # qom-list path=/machine/peripheral
3824     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3825     foreach my $per (@$resperipheral) {
3826         if ($per->{name} =~ m/^usb\d+$/) {
3827             $devices->{$per->{name}} = 1;
3828         }
3829     }
3830
3831     return $devices;
3832 }
3833
3834 sub vm_deviceplug {
3835     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3836
3837     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3838
3839     my $devices_list = vm_devices_list($vmid);
3840     return 1 if defined($devices_list->{$deviceid});
3841
3842     # add PCI bridge if we need it for the device
3843     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3844
3845     if ($deviceid eq 'tablet') {
3846
3847         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3848
3849     } elsif ($deviceid eq 'keyboard') {
3850
3851         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3852
3853     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3854
3855         die "usb hotplug currently not reliable\n";
3856         # since we can't reliably hot unplug all added usb devices and usb
3857         # passthrough breaks live migration we disable usb hotplugging for now
3858         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3859
3860     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3861
3862         qemu_iothread_add($vmid, $deviceid, $device);
3863
3864         qemu_driveadd($storecfg, $vmid, $device);
3865         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3866
3867         qemu_deviceadd($vmid, $devicefull);
3868         eval { qemu_deviceaddverify($vmid, $deviceid); };
3869         if (my $err = $@) {
3870             eval { qemu_drivedel($vmid, $deviceid); };
3871             warn $@ if $@;
3872             die $err;
3873         }
3874
3875     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3876
3877
3878         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3879         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3880         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3881
3882         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3883
3884         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3885             qemu_iothread_add($vmid, $deviceid, $device);
3886             $devicefull .= ",iothread=iothread-$deviceid";
3887         }
3888
3889         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3890             $devicefull .= ",num_queues=$device->{queues}";
3891         }
3892
3893         qemu_deviceadd($vmid, $devicefull);
3894         qemu_deviceaddverify($vmid, $deviceid);
3895
3896     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3897
3898         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3899         qemu_driveadd($storecfg, $vmid, $device);
3900
3901         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3902         eval { qemu_deviceadd($vmid, $devicefull); };
3903         if (my $err = $@) {
3904             eval { qemu_drivedel($vmid, $deviceid); };
3905             warn $@ if $@;
3906             die $err;
3907         }
3908
3909     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3910
3911         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3912
3913         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3914         my $use_old_bios_files = undef;
3915         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3916
3917         my $netdevicefull = print_netdevice_full(
3918             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3919         qemu_deviceadd($vmid, $netdevicefull);
3920         eval {
3921             qemu_deviceaddverify($vmid, $deviceid);
3922             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3923         };
3924         if (my $err = $@) {
3925             eval { qemu_netdevdel($vmid, $deviceid); };
3926             warn $@ if $@;
3927             die $err;
3928         }
3929
3930     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3931
3932         my $bridgeid = $2;
3933         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3934         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3935
3936         qemu_deviceadd($vmid, $devicefull);
3937         qemu_deviceaddverify($vmid, $deviceid);
3938
3939     } else {
3940         die "can't hotplug device '$deviceid'\n";
3941     }
3942
3943     return 1;
3944 }
3945
3946 # fixme: this should raise exceptions on error!
3947 sub vm_deviceunplug {
3948     my ($vmid, $conf, $deviceid) = @_;
3949
3950     my $devices_list = vm_devices_list($vmid);
3951     return 1 if !defined($devices_list->{$deviceid});
3952
3953     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3954     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3955
3956     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3957
3958         qemu_devicedel($vmid, $deviceid);
3959
3960     } elsif ($deviceid =~ m/^usb\d+$/) {
3961
3962         die "usb hotplug currently not reliable\n";
3963         # when unplugging usb devices this way, there may be remaining usb
3964         # controllers/hubs so we disable it for now
3965         #qemu_devicedel($vmid, $deviceid);
3966         #qemu_devicedelverify($vmid, $deviceid);
3967
3968     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3969
3970         qemu_devicedel($vmid, $deviceid);
3971         qemu_devicedelverify($vmid, $deviceid);
3972         qemu_drivedel($vmid, $deviceid);
3973         qemu_iothread_del($conf, $vmid, $deviceid);
3974
3975     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3976
3977         qemu_devicedel($vmid, $deviceid);
3978         qemu_devicedelverify($vmid, $deviceid);
3979         qemu_iothread_del($conf, $vmid, $deviceid);
3980
3981     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3982
3983         qemu_devicedel($vmid, $deviceid);
3984         qemu_drivedel($vmid, $deviceid);
3985         qemu_deletescsihw($conf, $vmid, $deviceid);
3986
3987     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3988
3989         qemu_devicedel($vmid, $deviceid);
3990         qemu_devicedelverify($vmid, $deviceid);
3991         qemu_netdevdel($vmid, $deviceid);
3992
3993     } else {
3994         die "can't unplug device '$deviceid'\n";
3995     }
3996
3997     return 1;
3998 }
3999
4000 sub qemu_deviceadd {
4001     my ($vmid, $devicefull) = @_;
4002
4003     $devicefull = "driver=".$devicefull;
4004     my %options =  split(/[=,]/, $devicefull);
4005
4006     mon_cmd($vmid, "device_add" , %options);
4007 }
4008
4009 sub qemu_devicedel {
4010     my ($vmid, $deviceid) = @_;
4011
4012     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4013 }
4014
4015 sub qemu_iothread_add {
4016     my($vmid, $deviceid, $device) = @_;
4017
4018     if ($device->{iothread}) {
4019         my $iothreads = vm_iothreads_list($vmid);
4020         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4021     }
4022 }
4023
4024 sub qemu_iothread_del {
4025     my($conf, $vmid, $deviceid) = @_;
4026
4027     my $confid = $deviceid;
4028     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4029         $confid = 'scsi' . $1;
4030     }
4031     my $device = parse_drive($confid, $conf->{$confid});
4032     if ($device->{iothread}) {
4033         my $iothreads = vm_iothreads_list($vmid);
4034         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4035     }
4036 }
4037
4038 sub qemu_objectadd {
4039     my($vmid, $objectid, $qomtype) = @_;
4040
4041     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4042
4043     return 1;
4044 }
4045
4046 sub qemu_objectdel {
4047     my($vmid, $objectid) = @_;
4048
4049     mon_cmd($vmid, "object-del", id => $objectid);
4050
4051     return 1;
4052 }
4053
4054 sub qemu_driveadd {
4055     my ($storecfg, $vmid, $device) = @_;
4056
4057     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4058     $drive =~ s/\\/\\\\/g;
4059     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4060
4061     # If the command succeeds qemu prints: "OK"
4062     return 1 if $ret =~ m/OK/s;
4063
4064     die "adding drive failed: $ret\n";
4065 }
4066
4067 sub qemu_drivedel {
4068     my($vmid, $deviceid) = @_;
4069
4070     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4071     $ret =~ s/^\s+//;
4072
4073     return 1 if $ret eq "";
4074
4075     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4076     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4077
4078     die "deleting drive $deviceid failed : $ret\n";
4079 }
4080
4081 sub qemu_deviceaddverify {
4082     my ($vmid, $deviceid) = @_;
4083
4084     for (my $i = 0; $i <= 5; $i++) {
4085          my $devices_list = vm_devices_list($vmid);
4086          return 1 if defined($devices_list->{$deviceid});
4087          sleep 1;
4088     }
4089
4090     die "error on hotplug device '$deviceid'\n";
4091 }
4092
4093
4094 sub qemu_devicedelverify {
4095     my ($vmid, $deviceid) = @_;
4096
4097     # need to verify that the device is correctly removed as device_del
4098     # is async and empty return is not reliable
4099
4100     for (my $i = 0; $i <= 5; $i++) {
4101          my $devices_list = vm_devices_list($vmid);
4102          return 1 if !defined($devices_list->{$deviceid});
4103          sleep 1;
4104     }
4105
4106     die "error on hot-unplugging device '$deviceid'\n";
4107 }
4108
4109 sub qemu_findorcreatescsihw {
4110     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4111
4112     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4113
4114     my $scsihwid="$controller_prefix$controller";
4115     my $devices_list = vm_devices_list($vmid);
4116
4117     if(!defined($devices_list->{$scsihwid})) {
4118         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4119     }
4120
4121     return 1;
4122 }
4123
4124 sub qemu_deletescsihw {
4125     my ($conf, $vmid, $opt) = @_;
4126
4127     my $device = parse_drive($opt, $conf->{$opt});
4128
4129     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4130         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4131         return 1;
4132     }
4133
4134     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4135
4136     my $devices_list = vm_devices_list($vmid);
4137     foreach my $opt (keys %{$devices_list}) {
4138         if (is_valid_drivename($opt)) {
4139             my $drive = parse_drive($opt, $conf->{$opt});
4140             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4141                 return 1;
4142             }
4143         }
4144     }
4145
4146     my $scsihwid="scsihw$controller";
4147
4148     vm_deviceunplug($vmid, $conf, $scsihwid);
4149
4150     return 1;
4151 }
4152
4153 sub qemu_add_pci_bridge {
4154     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4155
4156     my $bridges = {};
4157
4158     my $bridgeid;
4159
4160     print_pci_addr($device, $bridges, $arch, $machine_type);
4161
4162     while (my ($k, $v) = each %$bridges) {
4163         $bridgeid = $k;
4164     }
4165     return 1 if !defined($bridgeid) || $bridgeid < 1;
4166
4167     my $bridge = "pci.$bridgeid";
4168     my $devices_list = vm_devices_list($vmid);
4169
4170     if (!defined($devices_list->{$bridge})) {
4171         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4172     }
4173
4174     return 1;
4175 }
4176
4177 sub qemu_set_link_status {
4178     my ($vmid, $device, $up) = @_;
4179
4180     mon_cmd($vmid, "set_link", name => $device,
4181                up => $up ? JSON::true : JSON::false);
4182 }
4183
4184 sub qemu_netdevadd {
4185     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4186
4187     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4188     my %options =  split(/[=,]/, $netdev);
4189
4190     if (defined(my $vhost = $options{vhost})) {
4191         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4192     }
4193
4194     if (defined(my $queues = $options{queues})) {
4195         $options{queues} = $queues + 0;
4196     }
4197
4198     mon_cmd($vmid, "netdev_add",  %options);
4199     return 1;
4200 }
4201
4202 sub qemu_netdevdel {
4203     my ($vmid, $deviceid) = @_;
4204
4205     mon_cmd($vmid, "netdev_del", id => $deviceid);
4206 }
4207
4208 sub qemu_usb_hotplug {
4209     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4210
4211     return if !$device;
4212
4213     # remove the old one first
4214     vm_deviceunplug($vmid, $conf, $deviceid);
4215
4216     # check if xhci controller is necessary and available
4217     if ($device->{usb3}) {
4218
4219         my $devicelist = vm_devices_list($vmid);
4220
4221         if (!$devicelist->{xhci}) {
4222             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4223             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4224         }
4225     }
4226     my $d = parse_usb_device($device->{host});
4227     $d->{usb3} = $device->{usb3};
4228
4229     # add the new one
4230     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4231 }
4232
4233 sub qemu_cpu_hotplug {
4234     my ($vmid, $conf, $vcpus) = @_;
4235
4236     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4237
4238     my $sockets = 1;
4239     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4240     $sockets = $conf->{sockets} if  $conf->{sockets};
4241     my $cores = $conf->{cores} || 1;
4242     my $maxcpus = $sockets * $cores;
4243
4244     $vcpus = $maxcpus if !$vcpus;
4245
4246     die "you can't add more vcpus than maxcpus\n"
4247         if $vcpus > $maxcpus;
4248
4249     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4250
4251     if ($vcpus < $currentvcpus) {
4252
4253         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4254
4255             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4256                 qemu_devicedel($vmid, "cpu$i");
4257                 my $retry = 0;
4258                 my $currentrunningvcpus = undef;
4259                 while (1) {
4260                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4261                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4262                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4263                     $retry++;
4264                     sleep 1;
4265                 }
4266                 #update conf after each succesfull cpu unplug
4267                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4268                 PVE::QemuConfig->write_config($vmid, $conf);
4269             }
4270         } else {
4271             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4272         }
4273
4274         return;
4275     }
4276
4277     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4278     die "vcpus in running vm does not match its configuration\n"
4279         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4280
4281     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4282
4283         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4284             my $cpustr = print_cpu_device($conf, $i);
4285             qemu_deviceadd($vmid, $cpustr);
4286
4287             my $retry = 0;
4288             my $currentrunningvcpus = undef;
4289             while (1) {
4290                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4291                 last if scalar(@{$currentrunningvcpus}) == $i;
4292                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4293                 sleep 1;
4294                 $retry++;
4295             }
4296             #update conf after each succesfull cpu hotplug
4297             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4298             PVE::QemuConfig->write_config($vmid, $conf);
4299         }
4300     } else {
4301
4302         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4303             mon_cmd($vmid, "cpu-add", id => int($i));
4304         }
4305     }
4306 }
4307
4308 sub qemu_block_set_io_throttle {
4309     my ($vmid, $deviceid,
4310         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4311         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4312         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4313         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4314
4315     return if !check_running($vmid) ;
4316
4317     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4318         bps => int($bps),
4319         bps_rd => int($bps_rd),
4320         bps_wr => int($bps_wr),
4321         iops => int($iops),
4322         iops_rd => int($iops_rd),
4323         iops_wr => int($iops_wr),
4324         bps_max => int($bps_max),
4325         bps_rd_max => int($bps_rd_max),
4326         bps_wr_max => int($bps_wr_max),
4327         iops_max => int($iops_max),
4328         iops_rd_max => int($iops_rd_max),
4329         iops_wr_max => int($iops_wr_max),
4330         bps_max_length => int($bps_max_length),
4331         bps_rd_max_length => int($bps_rd_max_length),
4332         bps_wr_max_length => int($bps_wr_max_length),
4333         iops_max_length => int($iops_max_length),
4334         iops_rd_max_length => int($iops_rd_max_length),
4335         iops_wr_max_length => int($iops_wr_max_length),
4336     );
4337
4338 }
4339
4340 sub qemu_block_resize {
4341     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4342
4343     my $running = check_running($vmid);
4344
4345     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4346
4347     return if !$running;
4348
4349     my $padding = (1024 - $size % 1024) % 1024;
4350     $size = $size + $padding;
4351
4352     mon_cmd(
4353         $vmid,
4354         "block_resize",
4355         device => $deviceid,
4356         size => int($size),
4357         timeout => 60,
4358     );
4359 }
4360
4361 sub qemu_volume_snapshot {
4362     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4363
4364     my $running = check_running($vmid);
4365
4366     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4367         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4368     } else {
4369         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4370     }
4371 }
4372
4373 sub qemu_volume_snapshot_delete {
4374     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4375
4376     my $running = check_running($vmid);
4377
4378     if($running) {
4379
4380         $running = undef;
4381         my $conf = PVE::QemuConfig->load_config($vmid);
4382         PVE::QemuConfig->foreach_volume($conf, sub {
4383             my ($ds, $drive) = @_;
4384             $running = 1 if $drive->{file} eq $volid;
4385         });
4386     }
4387
4388     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4389         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4390     } else {
4391         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4392     }
4393 }
4394
4395 sub set_migration_caps {
4396     my ($vmid, $savevm) = @_;
4397
4398     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4399
4400     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4401     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4402
4403     my $cap_ref = [];
4404
4405     my $enabled_cap = {
4406         "auto-converge" => 1,
4407         "xbzrle" => 1,
4408         "x-rdma-pin-all" => 0,
4409         "zero-blocks" => 0,
4410         "compress" => 0,
4411         "dirty-bitmaps" => $dirty_bitmaps,
4412     };
4413
4414     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4415
4416     for my $supported_capability (@$supported_capabilities) {
4417         push @$cap_ref, {
4418             capability => $supported_capability->{capability},
4419             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4420         };
4421     }
4422
4423     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4424 }
4425
4426 sub foreach_volid {
4427     my ($conf, $func, @param) = @_;
4428
4429     my $volhash = {};
4430
4431     my $test_volid = sub {
4432         my ($key, $drive, $snapname) = @_;
4433
4434         my $volid = $drive->{file};
4435         return if !$volid;
4436
4437         $volhash->{$volid}->{cdrom} //= 1;
4438         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4439
4440         my $replicate = $drive->{replicate} // 1;
4441         $volhash->{$volid}->{replicate} //= 0;
4442         $volhash->{$volid}->{replicate} = 1 if $replicate;
4443
4444         $volhash->{$volid}->{shared} //= 0;
4445         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4446
4447         $volhash->{$volid}->{referenced_in_config} //= 0;
4448         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4449
4450         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4451             if defined($snapname);
4452
4453         my $size = $drive->{size};
4454         $volhash->{$volid}->{size} //= $size if $size;
4455
4456         $volhash->{$volid}->{is_vmstate} //= 0;
4457         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4458
4459         $volhash->{$volid}->{is_unused} //= 0;
4460         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4461     };
4462
4463     my $include_opts = {
4464         extra_keys => ['vmstate'],
4465         include_unused => 1,
4466     };
4467
4468     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4469     foreach my $snapname (keys %{$conf->{snapshots}}) {
4470         my $snap = $conf->{snapshots}->{$snapname};
4471         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4472     }
4473
4474     foreach my $volid (keys %$volhash) {
4475         &$func($volid, $volhash->{$volid}, @param);
4476     }
4477 }
4478
4479 my $fast_plug_option = {
4480     'lock' => 1,
4481     'name' => 1,
4482     'onboot' => 1,
4483     'shares' => 1,
4484     'startup' => 1,
4485     'description' => 1,
4486     'protection' => 1,
4487     'vmstatestorage' => 1,
4488     'hookscript' => 1,
4489     'tags' => 1,
4490 };
4491
4492 # hotplug changes in [PENDING]
4493 # $selection hash can be used to only apply specified options, for
4494 # example: { cores => 1 } (only apply changed 'cores')
4495 # $errors ref is used to return error messages
4496 sub vmconfig_hotplug_pending {
4497     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4498
4499     my $defaults = load_defaults();
4500     my $arch = get_vm_arch($conf);
4501     my $machine_type = get_vm_machine($conf, undef, $arch);
4502
4503     # commit values which do not have any impact on running VM first
4504     # Note: those option cannot raise errors, we we do not care about
4505     # $selection and always apply them.
4506
4507     my $add_error = sub {
4508         my ($opt, $msg) = @_;
4509         $errors->{$opt} = "hotplug problem - $msg";
4510     };
4511
4512     my $changes = 0;
4513     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4514         if ($fast_plug_option->{$opt}) {
4515             $conf->{$opt} = $conf->{pending}->{$opt};
4516             delete $conf->{pending}->{$opt};
4517             $changes = 1;
4518         }
4519     }
4520
4521     if ($changes) {
4522         PVE::QemuConfig->write_config($vmid, $conf);
4523     }
4524
4525     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4526
4527     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4528     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4529     foreach my $opt (sort keys %$pending_delete_hash) {
4530         next if $selection && !$selection->{$opt};
4531         my $force = $pending_delete_hash->{$opt}->{force};
4532         eval {
4533             if ($opt eq 'hotplug') {
4534                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4535             } elsif ($opt eq 'tablet') {
4536                 die "skip\n" if !$hotplug_features->{usb};
4537                 if ($defaults->{tablet}) {
4538                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4539                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4540                         if $arch eq 'aarch64';
4541                 } else {
4542                     vm_deviceunplug($vmid, $conf, 'tablet');
4543                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4544                 }
4545             } elsif ($opt =~ m/^usb\d+/) {
4546                 die "skip\n";
4547                 # since we cannot reliably hot unplug usb devices we are disabling it
4548                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4549                 #vm_deviceunplug($vmid, $conf, $opt);
4550             } elsif ($opt eq 'vcpus') {
4551                 die "skip\n" if !$hotplug_features->{cpu};
4552                 qemu_cpu_hotplug($vmid, $conf, undef);
4553             } elsif ($opt eq 'balloon') {
4554                 # enable balloon device is not hotpluggable
4555                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4556                 # here we reset the ballooning value to memory
4557                 my $balloon = $conf->{memory} || $defaults->{memory};
4558                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4559             } elsif ($fast_plug_option->{$opt}) {
4560                 # do nothing
4561             } elsif ($opt =~ m/^net(\d+)$/) {
4562                 die "skip\n" if !$hotplug_features->{network};
4563                 vm_deviceunplug($vmid, $conf, $opt);
4564             } elsif (is_valid_drivename($opt)) {
4565                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4566                 vm_deviceunplug($vmid, $conf, $opt);
4567                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4568             } elsif ($opt =~ m/^memory$/) {
4569                 die "skip\n" if !$hotplug_features->{memory};
4570                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4571             } elsif ($opt eq 'cpuunits') {
4572                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4573             } elsif ($opt eq 'cpulimit') {
4574                 $cgroup->change_cpu_quota(-1, 100000);
4575             } else {
4576                 die "skip\n";
4577             }
4578         };
4579         if (my $err = $@) {
4580             &$add_error($opt, $err) if $err ne "skip\n";
4581         } else {
4582             delete $conf->{$opt};
4583             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4584         }
4585     }
4586
4587     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4588     $apply_pending_cloudinit = sub {
4589         return if $apply_pending_cloudinit_done; # once is enough
4590         $apply_pending_cloudinit_done = 1; # once is enough
4591
4592         my ($key, $value) = @_;
4593
4594         my @cloudinit_opts = keys %$confdesc_cloudinit;
4595         foreach my $opt (keys %{$conf->{pending}}) {
4596             next if !grep { $_ eq $opt } @cloudinit_opts;
4597             $conf->{$opt} = delete $conf->{pending}->{$opt};
4598         }
4599
4600         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4601         foreach my $opt (sort keys %$pending_delete_hash) {
4602             next if !grep { $_ eq $opt } @cloudinit_opts;
4603             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4604             delete $conf->{$opt};
4605         }
4606
4607         my $new_conf = { %$conf };
4608         $new_conf->{$key} = $value;
4609         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4610     };
4611
4612     foreach my $opt (keys %{$conf->{pending}}) {
4613         next if $selection && !$selection->{$opt};
4614         my $value = $conf->{pending}->{$opt};
4615         eval {
4616             if ($opt eq 'hotplug') {
4617                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4618             } elsif ($opt eq 'tablet') {
4619                 die "skip\n" if !$hotplug_features->{usb};
4620                 if ($value == 1) {
4621                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4622                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4623                         if $arch eq 'aarch64';
4624                 } elsif ($value == 0) {
4625                     vm_deviceunplug($vmid, $conf, 'tablet');
4626                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4627                 }
4628             } elsif ($opt =~ m/^usb\d+$/) {
4629                 die "skip\n";
4630                 # since we cannot reliably hot unplug usb devices we disable it for now
4631                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4632                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4633                 #die "skip\n" if !$d;
4634                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4635             } elsif ($opt eq 'vcpus') {
4636                 die "skip\n" if !$hotplug_features->{cpu};
4637                 qemu_cpu_hotplug($vmid, $conf, $value);
4638             } elsif ($opt eq 'balloon') {
4639                 # enable/disable balloning device is not hotpluggable
4640                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4641                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4642                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4643
4644                 # allow manual ballooning if shares is set to zero
4645                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4646                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4647                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4648                 }
4649             } elsif ($opt =~ m/^net(\d+)$/) {
4650                 # some changes can be done without hotplug
4651                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4652                                     $vmid, $opt, $value, $arch, $machine_type);
4653             } elsif (is_valid_drivename($opt)) {
4654                 die "skip\n" if $opt eq 'efidisk0';
4655                 # some changes can be done without hotplug
4656                 my $drive = parse_drive($opt, $value);
4657                 if (drive_is_cloudinit($drive)) {
4658                     &$apply_pending_cloudinit($opt, $value);
4659                 }
4660                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4661                                      $vmid, $opt, $value, $arch, $machine_type);
4662             } elsif ($opt =~ m/^memory$/) { #dimms
4663                 die "skip\n" if !$hotplug_features->{memory};
4664                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4665             } elsif ($opt eq 'cpuunits') {
4666                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4667             } elsif ($opt eq 'cpulimit') {
4668                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4669                 $cgroup->change_cpu_quota($cpulimit, 100000);
4670             } else {
4671                 die "skip\n";  # skip non-hot-pluggable options
4672             }
4673         };
4674         if (my $err = $@) {
4675             &$add_error($opt, $err) if $err ne "skip\n";
4676         } else {
4677             $conf->{$opt} = $value;
4678             delete $conf->{pending}->{$opt};
4679         }
4680     }
4681
4682     PVE::QemuConfig->write_config($vmid, $conf);
4683 }
4684
4685 sub try_deallocate_drive {
4686     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4687
4688     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4689         my $volid = $drive->{file};
4690         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4691             my $sid = PVE::Storage::parse_volume_id($volid);
4692             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4693
4694             # check if the disk is really unused
4695             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4696                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4697             PVE::Storage::vdisk_free($storecfg, $volid);
4698             return 1;
4699         } else {
4700             # If vm is not owner of this disk remove from config
4701             return 1;
4702         }
4703     }
4704
4705     return;
4706 }
4707
4708 sub vmconfig_delete_or_detach_drive {
4709     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4710
4711     my $drive = parse_drive($opt, $conf->{$opt});
4712
4713     my $rpcenv = PVE::RPCEnvironment::get();
4714     my $authuser = $rpcenv->get_user();
4715
4716     if ($force) {
4717         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4718         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4719     } else {
4720         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4721     }
4722 }
4723
4724
4725
4726 sub vmconfig_apply_pending {
4727     my ($vmid, $conf, $storecfg, $errors) = @_;
4728
4729     my $add_apply_error = sub {
4730         my ($opt, $msg) = @_;
4731         my $err_msg = "unable to apply pending change $opt : $msg";
4732         $errors->{$opt} = $err_msg;
4733         warn $err_msg;
4734     };
4735
4736     # cold plug
4737
4738     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4739     foreach my $opt (sort keys %$pending_delete_hash) {
4740         my $force = $pending_delete_hash->{$opt}->{force};
4741         eval {
4742             if ($opt =~ m/^unused/) {
4743                 die "internal error";
4744             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4745                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4746             }
4747         };
4748         if (my $err = $@) {
4749             $add_apply_error->($opt, $err);
4750         } else {
4751             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4752             delete $conf->{$opt};
4753         }
4754     }
4755
4756     PVE::QemuConfig->cleanup_pending($conf);
4757
4758     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4759         next if $opt eq 'delete'; # just to be sure
4760         eval {
4761             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4762                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4763             }
4764         };
4765         if (my $err = $@) {
4766             $add_apply_error->($opt, $err);
4767         } else {
4768             $conf->{$opt} = delete $conf->{pending}->{$opt};
4769         }
4770     }
4771
4772     # write all changes at once to avoid unnecessary i/o
4773     PVE::QemuConfig->write_config($vmid, $conf);
4774 }
4775
4776 sub vmconfig_update_net {
4777     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4778
4779     my $newnet = parse_net($value);
4780
4781     if ($conf->{$opt}) {
4782         my $oldnet = parse_net($conf->{$opt});
4783
4784         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4785             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4786             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4787             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4788
4789             # for non online change, we try to hot-unplug
4790             die "skip\n" if !$hotplug;
4791             vm_deviceunplug($vmid, $conf, $opt);
4792         } else {
4793
4794             die "internal error" if $opt !~ m/net(\d+)/;
4795             my $iface = "tap${vmid}i$1";
4796
4797             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4798                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4799                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4800                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4801                 PVE::Network::tap_unplug($iface);
4802
4803                 if ($have_sdn) {
4804                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4805                 } else {
4806                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4807                 }
4808             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4809                 # Rate can be applied on its own but any change above needs to
4810                 # include the rate in tap_plug since OVS resets everything.
4811                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4812             }
4813
4814             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4815                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4816             }
4817
4818             return 1;
4819         }
4820     }
4821
4822     if ($hotplug) {
4823         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4824     } else {
4825         die "skip\n";
4826     }
4827 }
4828
4829 sub vmconfig_update_disk {
4830     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4831
4832     my $drive = parse_drive($opt, $value);
4833
4834     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4835         my $media = $drive->{media} || 'disk';
4836         my $oldmedia = $old_drive->{media} || 'disk';
4837         die "unable to change media type\n" if $media ne $oldmedia;
4838
4839         if (!drive_is_cdrom($old_drive)) {
4840
4841             if ($drive->{file} ne $old_drive->{file}) {
4842
4843                 die "skip\n" if !$hotplug;
4844
4845                 # unplug and register as unused
4846                 vm_deviceunplug($vmid, $conf, $opt);
4847                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4848
4849             } else {
4850                 # update existing disk
4851
4852                 # skip non hotpluggable value
4853                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4854                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4855                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4856                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4857                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4858                     die "skip\n";
4859                 }
4860
4861                 # apply throttle
4862                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4863                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4864                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4865                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4866                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4867                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4868                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4869                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4870                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4871                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4872                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4873                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4874                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4875                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4876                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4877                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4878                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4879                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4880
4881                     qemu_block_set_io_throttle(
4882                         $vmid,"drive-$opt",
4883                         ($drive->{mbps} || 0)*1024*1024,
4884                         ($drive->{mbps_rd} || 0)*1024*1024,
4885                         ($drive->{mbps_wr} || 0)*1024*1024,
4886                          $drive->{iops} || 0,
4887                          $drive->{iops_rd} || 0,
4888                          $drive->{iops_wr} || 0,
4889                         ($drive->{mbps_max} || 0)*1024*1024,
4890                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4891                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4892                          $drive->{iops_max} || 0,
4893                          $drive->{iops_rd_max} || 0,
4894                          $drive->{iops_wr_max} || 0,
4895                          $drive->{bps_max_length} || 1,
4896                          $drive->{bps_rd_max_length} || 1,
4897                          $drive->{bps_wr_max_length} || 1,
4898                          $drive->{iops_max_length} || 1,
4899                          $drive->{iops_rd_max_length} || 1,
4900                          $drive->{iops_wr_max_length} || 1,
4901                     );
4902
4903                 }
4904
4905                 return 1;
4906             }
4907
4908         } else { # cdrom
4909
4910             if ($drive->{file} eq 'none') {
4911                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4912                 if (drive_is_cloudinit($old_drive)) {
4913                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4914                 }
4915             } else {
4916                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4917
4918                 # force eject if locked
4919                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4920
4921                 if ($path) {
4922                     mon_cmd($vmid, "blockdev-change-medium",
4923                         id => "$opt", filename => "$path");
4924                 }
4925             }
4926
4927             return 1;
4928         }
4929     }
4930
4931     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4932     # hotplug new disks
4933     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4934     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4935 }
4936
4937 # called in locked context by incoming migration
4938 sub vm_migrate_get_nbd_disks {
4939     my ($storecfg, $conf, $replicated_volumes) = @_;
4940
4941     my $local_volumes = {};
4942     PVE::QemuConfig->foreach_volume($conf, sub {
4943         my ($ds, $drive) = @_;
4944
4945         return if drive_is_cdrom($drive);
4946
4947         my $volid = $drive->{file};
4948
4949         return if !$volid;
4950
4951         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4952
4953         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4954         return if $scfg->{shared};
4955
4956         # replicated disks re-use existing state via bitmap
4957         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4958         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4959     });
4960     return $local_volumes;
4961 }
4962
4963 # called in locked context by incoming migration
4964 sub vm_migrate_alloc_nbd_disks {
4965     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4966
4967     my $format = undef;
4968
4969     my $nbd = {};
4970     foreach my $opt (sort keys %$source_volumes) {
4971         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4972
4973         if ($use_existing) {
4974             $nbd->{$opt}->{drivestr} = print_drive($drive);
4975             $nbd->{$opt}->{volid} = $volid;
4976             $nbd->{$opt}->{replicated} = 1;
4977             next;
4978         }
4979
4980         # If a remote storage is specified and the format of the original
4981         # volume is not available there, fall back to the default format.
4982         # Otherwise use the same format as the original.
4983         if (!$storagemap->{identity}) {
4984             $storeid = map_storage($storagemap, $storeid);
4985             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4986             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4987             my $fileFormat = qemu_img_format($scfg, $volname);
4988             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4989         } else {
4990             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4991             $format = qemu_img_format($scfg, $volname);
4992         }
4993
4994         my $size = $drive->{size} / 1024;
4995         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4996         my $newdrive = $drive;
4997         $newdrive->{format} = $format;
4998         $newdrive->{file} = $newvolid;
4999         my $drivestr = print_drive($newdrive);
5000         $nbd->{$opt}->{drivestr} = $drivestr;
5001         $nbd->{$opt}->{volid} = $newvolid;
5002     }
5003
5004     return $nbd;
5005 }
5006
5007 # see vm_start_nolock for parameters, additionally:
5008 # migrate_opts:
5009 #   storagemap = parsed storage map for allocating NBD disks
5010 sub vm_start {
5011     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5012
5013     return PVE::QemuConfig->lock_config($vmid, sub {
5014         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5015
5016         die "you can't start a vm if it's a template\n"
5017             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5018
5019         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5020         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5021
5022         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5023
5024         if ($has_backup_lock && $running) {
5025             # a backup is currently running, attempt to start the guest in the
5026             # existing QEMU instance
5027             return vm_resume($vmid);
5028         }
5029
5030         PVE::QemuConfig->check_lock($conf)
5031             if !($params->{skiplock} || $has_suspended_lock);
5032
5033         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5034
5035         die "VM $vmid already running\n" if $running;
5036
5037         if (my $storagemap = $migrate_opts->{storagemap}) {
5038             my $replicated = $migrate_opts->{replicated_volumes};
5039             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5040             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5041
5042             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5043                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5044             }
5045         }
5046
5047         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5048     });
5049 }
5050
5051
5052 # params:
5053 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5054 #   skiplock => 0/1, skip checking for config lock
5055 #   skiptemplate => 0/1, skip checking whether VM is template
5056 #   forcemachine => to force Qemu machine (rollback/migration)
5057 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5058 #   timeout => in seconds
5059 #   paused => start VM in paused state (backup)
5060 #   resume => resume from hibernation
5061 #   pbs-backing => {
5062 #      sata0 => {
5063 #         repository
5064 #         snapshot
5065 #         keyfile
5066 #         archive
5067 #      },
5068 #      virtio2 => ...
5069 #   }
5070 # migrate_opts:
5071 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5072 #   migratedfrom => source node
5073 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5074 #   network => CIDR of migration network
5075 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5076 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5077 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5078 sub vm_start_nolock {
5079     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5080
5081     my $statefile = $params->{statefile};
5082     my $resume = $params->{resume};
5083
5084     my $migratedfrom = $migrate_opts->{migratedfrom};
5085     my $migration_type = $migrate_opts->{type};
5086
5087     my $res = {};
5088
5089     # clean up leftover reboot request files
5090     eval { clear_reboot_request($vmid); };
5091     warn $@ if $@;
5092
5093     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5094         vmconfig_apply_pending($vmid, $conf, $storecfg);
5095         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5096     }
5097
5098     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5099
5100     my $defaults = load_defaults();
5101
5102     # set environment variable useful inside network script
5103     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5104
5105     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5106
5107     my $forcemachine = $params->{forcemachine};
5108     my $forcecpu = $params->{forcecpu};
5109     if ($resume) {
5110         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5111         $forcemachine = $conf->{runningmachine};
5112         $forcecpu = $conf->{runningcpu};
5113         print "Resuming suspended VM\n";
5114     }
5115
5116     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5117         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5118
5119     my $migration_ip;
5120     my $get_migration_ip = sub {
5121         my ($nodename) = @_;
5122
5123         return $migration_ip if defined($migration_ip);
5124
5125         my $cidr = $migrate_opts->{network};
5126
5127         if (!defined($cidr)) {
5128             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5129             $cidr = $dc_conf->{migration}->{network};
5130         }
5131
5132         if (defined($cidr)) {
5133             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5134
5135             die "could not get IP: no address configured on local " .
5136                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5137
5138             die "could not get IP: multiple addresses configured on local " .
5139                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5140
5141             $migration_ip = @$ips[0];
5142         }
5143
5144         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5145             if !defined($migration_ip);
5146
5147         return $migration_ip;
5148     };
5149
5150     my $migrate_uri;
5151     if ($statefile) {
5152         if ($statefile eq 'tcp') {
5153             my $localip = "localhost";
5154             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5155             my $nodename = nodename();
5156
5157             if (!defined($migration_type)) {
5158                 if (defined($datacenterconf->{migration}->{type})) {
5159                     $migration_type = $datacenterconf->{migration}->{type};
5160                 } else {
5161                     $migration_type = 'secure';
5162                 }
5163             }
5164
5165             if ($migration_type eq 'insecure') {
5166                 $localip = $get_migration_ip->($nodename);
5167                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5168             }
5169
5170             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5171             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5172             $migrate_uri = "tcp:${localip}:${migrate_port}";
5173             push @$cmd, '-incoming', $migrate_uri;
5174             push @$cmd, '-S';
5175
5176         } elsif ($statefile eq 'unix') {
5177             # should be default for secure migrations as a ssh TCP forward
5178             # tunnel is not deterministic reliable ready and fails regurarly
5179             # to set up in time, so use UNIX socket forwards
5180             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5181             unlink $socket_addr;
5182
5183             $migrate_uri = "unix:$socket_addr";
5184
5185             push @$cmd, '-incoming', $migrate_uri;
5186             push @$cmd, '-S';
5187
5188         } elsif (-e $statefile) {
5189             push @$cmd, '-loadstate', $statefile;
5190         } else {
5191             my $statepath = PVE::Storage::path($storecfg, $statefile);
5192             push @$vollist, $statefile;
5193             push @$cmd, '-loadstate', $statepath;
5194         }
5195     } elsif ($params->{paused}) {
5196         push @$cmd, '-S';
5197     }
5198
5199     # host pci devices
5200     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5201       my $d = parse_hostpci($conf->{"hostpci$i"});
5202       next if !$d;
5203       my $pcidevices = $d->{pciid};
5204       foreach my $pcidevice (@$pcidevices) {
5205             my $pciid = $pcidevice->{id};
5206
5207             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5208             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5209             die "no pci device info for device '$pciid'\n" if !$info;
5210
5211             if ($d->{mdev}) {
5212                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5213                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5214             } else {
5215                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5216                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5217                 die "can't reset PCI device '$pciid'\n"
5218                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5219             }
5220       }
5221     }
5222
5223     PVE::Storage::activate_volumes($storecfg, $vollist);
5224
5225     eval {
5226         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5227             outfunc => sub {}, errfunc => sub {});
5228     };
5229     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5230     # timeout should be more than enough here...
5231     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5232
5233     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5234                                               : $defaults->{cpuunits};
5235
5236     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5237     my %run_params = (
5238         timeout => $statefile ? undef : $start_timeout,
5239         umask => 0077,
5240         noerr => 1,
5241     );
5242
5243     # when migrating, prefix QEMU output so other side can pick up any
5244     # errors that might occur and show the user
5245     if ($migratedfrom) {
5246         $run_params{quiet} = 1;
5247         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5248     }
5249
5250     my %properties = (
5251         Slice => 'qemu.slice',
5252         KillMode => 'none'
5253     );
5254
5255     if (PVE::CGroup::cgroup_mode() == 2) {
5256         $properties{CPUWeight} = $cpuunits;
5257     } else {
5258         $properties{CPUShares} = $cpuunits;
5259     }
5260
5261     if (my $cpulimit = $conf->{cpulimit}) {
5262         $properties{CPUQuota} = int($cpulimit * 100);
5263     }
5264     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5265
5266     my $run_qemu = sub {
5267         PVE::Tools::run_fork sub {
5268             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5269
5270             my $exitcode = run_command($cmd, %run_params);
5271             die "QEMU exited with code $exitcode\n" if $exitcode;
5272         };
5273     };
5274
5275     if ($conf->{hugepages}) {
5276
5277         my $code = sub {
5278             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5279             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5280
5281             PVE::QemuServer::Memory::hugepages_mount();
5282             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5283
5284             eval { $run_qemu->() };
5285             if (my $err = $@) {
5286                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5287                     if !$conf->{keephugepages};
5288                 die $err;
5289             }
5290
5291             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5292                 if !$conf->{keephugepages};
5293         };
5294         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5295
5296     } else {
5297         eval { $run_qemu->() };
5298     }
5299
5300     if (my $err = $@) {
5301         # deactivate volumes if start fails
5302         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5303         die "start failed: $err";
5304     }
5305
5306     print "migration listens on $migrate_uri\n" if $migrate_uri;
5307     $res->{migrate_uri} = $migrate_uri;
5308
5309     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5310         eval { mon_cmd($vmid, "cont"); };
5311         warn $@ if $@;
5312     }
5313
5314     #start nbd server for storage migration
5315     if (my $nbd = $migrate_opts->{nbd}) {
5316         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5317
5318         my $migrate_storage_uri;
5319         # nbd_protocol_version > 0 for unix socket support
5320         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5321             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5322             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5323             $migrate_storage_uri = "nbd:unix:$socket_path";
5324         } else {
5325             my $nodename = nodename();
5326             my $localip = $get_migration_ip->($nodename);
5327             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5328             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5329
5330             mon_cmd($vmid, "nbd-server-start", addr => {
5331                 type => 'inet',
5332                 data => {
5333                     host => "${localip}",
5334                     port => "${storage_migrate_port}",
5335                 },
5336             });
5337             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5338             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5339         }
5340
5341         $res->{migrate_storage_uri} = $migrate_storage_uri;
5342
5343         foreach my $opt (sort keys %$nbd) {
5344             my $drivestr = $nbd->{$opt}->{drivestr};
5345             my $volid = $nbd->{$opt}->{volid};
5346             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5347             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5348             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5349             print "re-using replicated volume: $opt - $volid\n"
5350                 if $nbd->{$opt}->{replicated};
5351
5352             $res->{drives}->{$opt} = $nbd->{$opt};
5353             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5354         }
5355     }
5356
5357     if ($migratedfrom) {
5358         eval {
5359             set_migration_caps($vmid);
5360         };
5361         warn $@ if $@;
5362
5363         if ($spice_port) {
5364             print "spice listens on port $spice_port\n";
5365             $res->{spice_port} = $spice_port;
5366             if ($migrate_opts->{spice_ticket}) {
5367                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5368                     $migrate_opts->{spice_ticket});
5369                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5370             }
5371         }
5372
5373     } else {
5374         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5375             if !$statefile && $conf->{balloon};
5376
5377         foreach my $opt (keys %$conf) {
5378             next if $opt !~  m/^net\d+$/;
5379             my $nicconf = parse_net($conf->{$opt});
5380             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5381         }
5382     }
5383
5384     mon_cmd($vmid, 'qom-set',
5385                 path => "machine/peripheral/balloon0",
5386                 property => "guest-stats-polling-interval",
5387                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5388
5389     if ($resume) {
5390         print "Resumed VM, removing state\n";
5391         if (my $vmstate = $conf->{vmstate}) {
5392             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5393             PVE::Storage::vdisk_free($storecfg, $vmstate);
5394         }
5395         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5396         PVE::QemuConfig->write_config($vmid, $conf);
5397     }
5398
5399     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5400
5401     return $res;
5402 }
5403
5404 sub vm_commandline {
5405     my ($storecfg, $vmid, $snapname) = @_;
5406
5407     my $conf = PVE::QemuConfig->load_config($vmid);
5408     my $forcemachine;
5409     my $forcecpu;
5410
5411     if ($snapname) {
5412         my $snapshot = $conf->{snapshots}->{$snapname};
5413         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5414
5415         # check for machine or CPU overrides in snapshot
5416         $forcemachine = $snapshot->{runningmachine};
5417         $forcecpu = $snapshot->{runningcpu};
5418
5419         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5420
5421         $conf = $snapshot;
5422     }
5423
5424     my $defaults = load_defaults();
5425
5426     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5427         $forcemachine, $forcecpu);
5428
5429     return PVE::Tools::cmd2string($cmd);
5430 }
5431
5432 sub vm_reset {
5433     my ($vmid, $skiplock) = @_;
5434
5435     PVE::QemuConfig->lock_config($vmid, sub {
5436
5437         my $conf = PVE::QemuConfig->load_config($vmid);
5438
5439         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5440
5441         mon_cmd($vmid, "system_reset");
5442     });
5443 }
5444
5445 sub get_vm_volumes {
5446     my ($conf) = @_;
5447
5448     my $vollist = [];
5449     foreach_volid($conf, sub {
5450         my ($volid, $attr) = @_;
5451
5452         return if $volid =~ m|^/|;
5453
5454         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5455         return if !$sid;
5456
5457         push @$vollist, $volid;
5458     });
5459
5460     return $vollist;
5461 }
5462
5463 sub vm_stop_cleanup {
5464     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5465
5466     eval {
5467
5468         if (!$keepActive) {
5469             my $vollist = get_vm_volumes($conf);
5470             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5471         }
5472
5473         foreach my $ext (qw(mon qmp pid vnc qga)) {
5474             unlink "/var/run/qemu-server/${vmid}.$ext";
5475         }
5476
5477         if ($conf->{ivshmem}) {
5478             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5479             # just delete it for now, VMs which have this already open do not
5480             # are affected, but new VMs will get a separated one. If this
5481             # becomes an issue we either add some sort of ref-counting or just
5482             # add a "don't delete on stop" flag to the ivshmem format.
5483             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5484         }
5485
5486         foreach my $key (keys %$conf) {
5487             next if $key !~ m/^hostpci(\d+)$/;
5488             my $hostpciindex = $1;
5489             my $d = parse_hostpci($conf->{$key});
5490             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5491
5492             foreach my $pci (@{$d->{pciid}}) {
5493                 my $pciid = $pci->{id};
5494                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5495             }
5496         }
5497
5498         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5499     };
5500     warn $@ if $@; # avoid errors - just warn
5501 }
5502
5503 # call only in locked context
5504 sub _do_vm_stop {
5505     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5506
5507     my $pid = check_running($vmid, $nocheck);
5508     return if !$pid;
5509
5510     my $conf;
5511     if (!$nocheck) {
5512         $conf = PVE::QemuConfig->load_config($vmid);
5513         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5514         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5515             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5516             $timeout = $opts->{down} if $opts->{down};
5517         }
5518         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5519     }
5520
5521     eval {
5522         if ($shutdown) {
5523             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5524                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5525             } else {
5526                 mon_cmd($vmid, "system_powerdown");
5527             }
5528         } else {
5529             mon_cmd($vmid, "quit");
5530         }
5531     };
5532     my $err = $@;
5533
5534     if (!$err) {
5535         $timeout = 60 if !defined($timeout);
5536
5537         my $count = 0;
5538         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5539             $count++;
5540             sleep 1;
5541         }
5542
5543         if ($count >= $timeout) {
5544             if ($force) {
5545                 warn "VM still running - terminating now with SIGTERM\n";
5546                 kill 15, $pid;
5547             } else {
5548                 die "VM quit/powerdown failed - got timeout\n";
5549             }
5550         } else {
5551             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5552             return;
5553         }
5554     } else {
5555        if (!check_running($vmid, $nocheck)) {
5556             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5557             return;
5558        }
5559        if ($force) {
5560             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5561             kill 15, $pid;
5562         } else {
5563             die "VM quit/powerdown failed\n";
5564         }
5565     }
5566
5567     # wait again
5568     $timeout = 10;
5569
5570     my $count = 0;
5571     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5572         $count++;
5573         sleep 1;
5574     }
5575
5576     if ($count >= $timeout) {
5577         warn "VM still running - terminating now with SIGKILL\n";
5578         kill 9, $pid;
5579         sleep 1;
5580     }
5581
5582     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5583 }
5584
5585 # Note: use $nocheck to skip tests if VM configuration file exists.
5586 # We need that when migration VMs to other nodes (files already moved)
5587 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5588 sub vm_stop {
5589     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5590
5591     $force = 1 if !defined($force) && !$shutdown;
5592
5593     if ($migratedfrom){
5594         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5595         kill 15, $pid if $pid;
5596         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5597         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5598         return;
5599     }
5600
5601     PVE::QemuConfig->lock_config($vmid, sub {
5602         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5603    });
5604 }
5605
5606 sub vm_reboot {
5607     my ($vmid, $timeout) = @_;
5608
5609     PVE::QemuConfig->lock_config($vmid, sub {
5610         eval {
5611
5612             # only reboot if running, as qmeventd starts it again on a stop event
5613             return if !check_running($vmid);
5614
5615             create_reboot_request($vmid);
5616
5617             my $storecfg = PVE::Storage::config();
5618             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5619
5620         };
5621         if (my $err = $@) {
5622             # avoid that the next normal shutdown will be confused for a reboot
5623             clear_reboot_request($vmid);
5624             die $err;
5625         }
5626    });
5627 }
5628
5629 # note: if using the statestorage parameter, the caller has to check privileges
5630 sub vm_suspend {
5631     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5632
5633     my $conf;
5634     my $path;
5635     my $storecfg;
5636     my $vmstate;
5637
5638     PVE::QemuConfig->lock_config($vmid, sub {
5639
5640         $conf = PVE::QemuConfig->load_config($vmid);
5641
5642         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5643         PVE::QemuConfig->check_lock($conf)
5644             if !($skiplock || $is_backing_up);
5645
5646         die "cannot suspend to disk during backup\n"
5647             if $is_backing_up && $includestate;
5648
5649         if ($includestate) {
5650             $conf->{lock} = 'suspending';
5651             my $date = strftime("%Y-%m-%d", localtime(time()));
5652             $storecfg = PVE::Storage::config();
5653             if (!$statestorage) {
5654                 $statestorage = find_vmstate_storage($conf, $storecfg);
5655                 # check permissions for the storage
5656                 my $rpcenv = PVE::RPCEnvironment::get();
5657                 if ($rpcenv->{type} ne 'cli') {
5658                     my $authuser = $rpcenv->get_user();
5659                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5660                 }
5661             }
5662
5663
5664             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5665                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5666             $path = PVE::Storage::path($storecfg, $vmstate);
5667             PVE::QemuConfig->write_config($vmid, $conf);
5668         } else {
5669             mon_cmd($vmid, "stop");
5670         }
5671     });
5672
5673     if ($includestate) {
5674         # save vm state
5675         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5676
5677         eval {
5678             set_migration_caps($vmid, 1);
5679             mon_cmd($vmid, "savevm-start", statefile => $path);
5680             for(;;) {
5681                 my $state = mon_cmd($vmid, "query-savevm");
5682                 if (!$state->{status}) {
5683                     die "savevm not active\n";
5684                 } elsif ($state->{status} eq 'active') {
5685                     sleep(1);
5686                     next;
5687                 } elsif ($state->{status} eq 'completed') {
5688                     print "State saved, quitting\n";
5689                     last;
5690                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5691                     die "query-savevm failed with error '$state->{error}'\n"
5692                 } else {
5693                     die "query-savevm returned status '$state->{status}'\n";
5694                 }
5695             }
5696         };
5697         my $err = $@;
5698
5699         PVE::QemuConfig->lock_config($vmid, sub {
5700             $conf = PVE::QemuConfig->load_config($vmid);
5701             if ($err) {
5702                 # cleanup, but leave suspending lock, to indicate something went wrong
5703                 eval {
5704                     mon_cmd($vmid, "savevm-end");
5705                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5706                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5707                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5708                     PVE::QemuConfig->write_config($vmid, $conf);
5709                 };
5710                 warn $@ if $@;
5711                 die $err;
5712             }
5713
5714             die "lock changed unexpectedly\n"
5715                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5716
5717             mon_cmd($vmid, "quit");
5718             $conf->{lock} = 'suspended';
5719             PVE::QemuConfig->write_config($vmid, $conf);
5720         });
5721     }
5722 }
5723
5724 sub vm_resume {
5725     my ($vmid, $skiplock, $nocheck) = @_;
5726
5727     PVE::QemuConfig->lock_config($vmid, sub {
5728         my $res = mon_cmd($vmid, 'query-status');
5729         my $resume_cmd = 'cont';
5730         my $reset = 0;
5731
5732         if ($res->{status}) {
5733             return if $res->{status} eq 'running'; # job done, go home
5734             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5735             $reset = 1 if $res->{status} eq 'shutdown';
5736         }
5737
5738         if (!$nocheck) {
5739
5740             my $conf = PVE::QemuConfig->load_config($vmid);
5741
5742             PVE::QemuConfig->check_lock($conf)
5743                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5744         }
5745
5746         if ($reset) {
5747             # required if a VM shuts down during a backup and we get a resume
5748             # request before the backup finishes for example
5749             mon_cmd($vmid, "system_reset");
5750         }
5751         mon_cmd($vmid, $resume_cmd);
5752     });
5753 }
5754
5755 sub vm_sendkey {
5756     my ($vmid, $skiplock, $key) = @_;
5757
5758     PVE::QemuConfig->lock_config($vmid, sub {
5759
5760         my $conf = PVE::QemuConfig->load_config($vmid);
5761
5762         # there is no qmp command, so we use the human monitor command
5763         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5764         die $res if $res ne '';
5765     });
5766 }
5767
5768 # vzdump restore implementaion
5769
5770 sub tar_archive_read_firstfile {
5771     my $archive = shift;
5772
5773     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5774
5775     # try to detect archive type first
5776     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5777         die "unable to open file '$archive'\n";
5778     my $firstfile = <$fh>;
5779     kill 15, $pid;
5780     close $fh;
5781
5782     die "ERROR: archive contaions no data\n" if !$firstfile;
5783     chomp $firstfile;
5784
5785     return $firstfile;
5786 }
5787
5788 sub tar_restore_cleanup {
5789     my ($storecfg, $statfile) = @_;
5790
5791     print STDERR "starting cleanup\n";
5792
5793     if (my $fd = IO::File->new($statfile, "r")) {
5794         while (defined(my $line = <$fd>)) {
5795             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5796                 my $volid = $2;
5797                 eval {
5798                     if ($volid =~ m|^/|) {
5799                         unlink $volid || die 'unlink failed\n';
5800                     } else {
5801                         PVE::Storage::vdisk_free($storecfg, $volid);
5802                     }
5803                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5804                 };
5805                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5806             } else {
5807                 print STDERR "unable to parse line in statfile - $line";
5808             }
5809         }
5810         $fd->close();
5811     }
5812 }
5813
5814 sub restore_file_archive {
5815     my ($archive, $vmid, $user, $opts) = @_;
5816
5817     return restore_vma_archive($archive, $vmid, $user, $opts)
5818         if $archive eq '-';
5819
5820     my $info = PVE::Storage::archive_info($archive);
5821     my $format = $opts->{format} // $info->{format};
5822     my $comp = $info->{compression};
5823
5824     # try to detect archive format
5825     if ($format eq 'tar') {
5826         return restore_tar_archive($archive, $vmid, $user, $opts);
5827     } else {
5828         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5829     }
5830 }
5831
5832 # hepler to remove disks that will not be used after restore
5833 my $restore_cleanup_oldconf = sub {
5834     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5835
5836     PVE::QemuConfig->foreach_volume($oldconf, sub {
5837         my ($ds, $drive) = @_;
5838
5839         return if drive_is_cdrom($drive, 1);
5840
5841         my $volid = $drive->{file};
5842         return if !$volid || $volid =~ m|^/|;
5843
5844         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5845         return if !$path || !$owner || ($owner != $vmid);
5846
5847         # Note: only delete disk we want to restore
5848         # other volumes will become unused
5849         if ($virtdev_hash->{$ds}) {
5850             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5851             if (my $err = $@) {
5852                 warn $err;
5853             }
5854         }
5855     });
5856
5857     # delete vmstate files, after the restore we have no snapshots anymore
5858     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5859         my $snap = $oldconf->{snapshots}->{$snapname};
5860         if ($snap->{vmstate}) {
5861             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5862             if (my $err = $@) {
5863                 warn $err;
5864             }
5865         }
5866     }
5867 };
5868
5869 # Helper to parse vzdump backup device hints
5870 #
5871 # $rpcenv: Environment, used to ckeck storage permissions
5872 # $user: User ID, to check storage permissions
5873 # $storecfg: Storage configuration
5874 # $fh: the file handle for reading the configuration
5875 # $devinfo: should contain device sizes for all backu-up'ed devices
5876 # $options: backup options (pool, default storage)
5877 #
5878 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5879 my $parse_backup_hints = sub {
5880     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5881
5882     my $virtdev_hash = {};
5883
5884     while (defined(my $line = <$fh>)) {
5885         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5886             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5887             die "archive does not contain data for drive '$virtdev'\n"
5888                 if !$devinfo->{$devname};
5889
5890             if (defined($options->{storage})) {
5891                 $storeid = $options->{storage} || 'local';
5892             } elsif (!$storeid) {
5893                 $storeid = 'local';
5894             }
5895             $format = 'raw' if !$format;
5896             $devinfo->{$devname}->{devname} = $devname;
5897             $devinfo->{$devname}->{virtdev} = $virtdev;
5898             $devinfo->{$devname}->{format} = $format;
5899             $devinfo->{$devname}->{storeid} = $storeid;
5900
5901             # check permission on storage
5902             my $pool = $options->{pool}; # todo: do we need that?
5903             if ($user ne 'root@pam') {
5904                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5905             }
5906
5907             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5908         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5909             my $virtdev = $1;
5910             my $drive = parse_drive($virtdev, $2);
5911             if (drive_is_cloudinit($drive)) {
5912                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5913                 $storeid = $options->{storage} if defined ($options->{storage});
5914                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5915                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5916
5917                 $virtdev_hash->{$virtdev} = {
5918                     format => $format,
5919                     storeid => $storeid,
5920                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5921                     is_cloudinit => 1,
5922                 };
5923             }
5924         }
5925     }
5926
5927     return $virtdev_hash;
5928 };
5929
5930 # Helper to allocate and activate all volumes required for a restore
5931 #
5932 # $storecfg: Storage configuration
5933 # $virtdev_hash: as returned by parse_backup_hints()
5934 #
5935 # Returns: { $virtdev => $volid }
5936 my $restore_allocate_devices = sub {
5937     my ($storecfg, $virtdev_hash, $vmid) = @_;
5938
5939     my $map = {};
5940     foreach my $virtdev (sort keys %$virtdev_hash) {
5941         my $d = $virtdev_hash->{$virtdev};
5942         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5943         my $storeid = $d->{storeid};
5944         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5945
5946         # test if requested format is supported
5947         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5948         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5949         $d->{format} = $defFormat if !$supported;
5950
5951         my $name;
5952         if ($d->{is_cloudinit}) {
5953             $name = "vm-$vmid-cloudinit";
5954             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5955             if ($scfg->{path}) {
5956                 $name .= ".$d->{format}";
5957             }
5958         }
5959
5960         my $volid = PVE::Storage::vdisk_alloc(
5961             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5962
5963         print STDERR "new volume ID is '$volid'\n";
5964         $d->{volid} = $volid;
5965
5966         PVE::Storage::activate_volumes($storecfg, [$volid]);
5967
5968         $map->{$virtdev} = $volid;
5969     }
5970
5971     return $map;
5972 };
5973
5974 sub restore_update_config_line {
5975     my ($cookie, $map, $line, $unique) = @_;
5976
5977     return '' if $line =~ m/^\#qmdump\#/;
5978     return '' if $line =~ m/^\#vzdump\#/;
5979     return '' if $line =~ m/^lock:/;
5980     return '' if $line =~ m/^unused\d+:/;
5981     return '' if $line =~ m/^parent:/;
5982
5983     my $res = '';
5984
5985     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5986     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5987         # try to convert old 1.X settings
5988         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5989         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5990             my ($model, $macaddr) = split(/\=/, $devconfig);
5991             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5992             my $net = {
5993                 model => $model,
5994                 bridge => "vmbr$ind",
5995                 macaddr => $macaddr,
5996             };
5997             my $netstr = print_net($net);
5998
5999             $res .= "net$cookie->{netcount}: $netstr\n";
6000             $cookie->{netcount}++;
6001         }
6002     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6003         my ($id, $netstr) = ($1, $2);
6004         my $net = parse_net($netstr);
6005         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6006         $netstr = print_net($net);
6007         $res .= "$id: $netstr\n";
6008     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6009         my $virtdev = $1;
6010         my $value = $3;
6011         my $di = parse_drive($virtdev, $value);
6012         if (defined($di->{backup}) && !$di->{backup}) {
6013             $res .= "#$line";
6014         } elsif ($map->{$virtdev}) {
6015             delete $di->{format}; # format can change on restore
6016             $di->{file} = $map->{$virtdev};
6017             $value = print_drive($di);
6018             $res .= "$virtdev: $value\n";
6019         } else {
6020             $res .= $line;
6021         }
6022     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6023         my $vmgenid = $1;
6024         if ($vmgenid ne '0') {
6025             # always generate a new vmgenid if there was a valid one setup
6026             $vmgenid = generate_uuid();
6027         }
6028         $res .= "vmgenid: $vmgenid\n";
6029     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6030         my ($uuid, $uuid_str);
6031         UUID::generate($uuid);
6032         UUID::unparse($uuid, $uuid_str);
6033         my $smbios1 = parse_smbios1($2);
6034         $smbios1->{uuid} = $uuid_str;
6035         $res .= $1.print_smbios1($smbios1)."\n";
6036     } else {
6037         $res .= $line;
6038     }
6039
6040     return $res;
6041 }
6042
6043 my $restore_deactivate_volumes = sub {
6044     my ($storecfg, $devinfo) = @_;
6045
6046     my $vollist = [];
6047     foreach my $devname (keys %$devinfo) {
6048         my $volid = $devinfo->{$devname}->{volid};
6049         push @$vollist, $volid if $volid;
6050     }
6051
6052     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6053 };
6054
6055 my $restore_destroy_volumes = sub {
6056     my ($storecfg, $devinfo) = @_;
6057
6058     foreach my $devname (keys %$devinfo) {
6059         my $volid = $devinfo->{$devname}->{volid};
6060         next if !$volid;
6061         eval {
6062             if ($volid =~ m|^/|) {
6063                 unlink $volid || die 'unlink failed\n';
6064             } else {
6065                 PVE::Storage::vdisk_free($storecfg, $volid);
6066             }
6067             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6068         };
6069         print STDERR "unable to cleanup '$volid' - $@" if $@;
6070     }
6071 };
6072
6073 # FIXME For PVE 7.0, remove $content_type and always use 'images'
6074 sub scan_volids {
6075     my ($cfg, $vmid, $content_type) = @_;
6076
6077     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, $content_type);
6078
6079     my $volid_hash = {};
6080     foreach my $storeid (keys %$info) {
6081         foreach my $item (@{$info->{$storeid}}) {
6082             next if !($item->{volid} && $item->{size});
6083             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6084             $volid_hash->{$item->{volid}} = $item;
6085         }
6086     }
6087
6088     return $volid_hash;
6089 }
6090
6091 sub update_disk_config {
6092     my ($vmid, $conf, $volid_hash) = @_;
6093
6094     my $changes;
6095     my $prefix = "VM $vmid";
6096
6097     # used and unused disks
6098     my $referenced = {};
6099
6100     # Note: it is allowed to define multiple storages with same path (alias), so
6101     # we need to check both 'volid' and real 'path' (two different volid can point
6102     # to the same path).
6103
6104     my $referencedpath = {};
6105
6106     # update size info
6107     PVE::QemuConfig->foreach_volume($conf, sub {
6108         my ($opt, $drive) = @_;
6109
6110         my $volid = $drive->{file};
6111         return if !$volid;
6112         my $volume = $volid_hash->{$volid};
6113
6114         # mark volid as "in-use" for next step
6115         $referenced->{$volid} = 1;
6116         if ($volume && (my $path = $volume->{path})) {
6117             $referencedpath->{$path} = 1;
6118         }
6119
6120         return if drive_is_cdrom($drive);
6121         return if !$volume;
6122
6123         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6124         if (defined($updated)) {
6125             $changes = 1;
6126             $conf->{$opt} = print_drive($updated);
6127             print "$prefix ($opt): $msg\n";
6128         }
6129     });
6130
6131     # remove 'unusedX' entry if volume is used
6132     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6133         my ($opt, $drive) = @_;
6134
6135         my $volid = $drive->{file};
6136         return if !$volid;
6137
6138         my $path;
6139         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6140         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6141             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6142             $changes = 1;
6143             delete $conf->{$opt};
6144         }
6145
6146         $referenced->{$volid} = 1;
6147         $referencedpath->{$path} = 1 if $path;
6148     });
6149
6150     foreach my $volid (sort keys %$volid_hash) {
6151         next if $volid =~ m/vm-$vmid-state-/;
6152         next if $referenced->{$volid};
6153         my $path = $volid_hash->{$volid}->{path};
6154         next if !$path; # just to be sure
6155         next if $referencedpath->{$path};
6156         $changes = 1;
6157         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6158         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6159         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6160     }
6161
6162     return $changes;
6163 }
6164
6165 sub rescan {
6166     my ($vmid, $nolock, $dryrun) = @_;
6167
6168     my $cfg = PVE::Storage::config();
6169
6170     print "rescan volumes...\n";
6171     my $volid_hash = scan_volids($cfg, $vmid, 'images');
6172
6173     my $updatefn =  sub {
6174         my ($vmid) = @_;
6175
6176         my $conf = PVE::QemuConfig->load_config($vmid);
6177
6178         PVE::QemuConfig->check_lock($conf);
6179
6180         my $vm_volids = {};
6181         foreach my $volid (keys %$volid_hash) {
6182             my $info = $volid_hash->{$volid};
6183             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6184         }
6185
6186         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6187
6188         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6189     };
6190
6191     if (defined($vmid)) {
6192         if ($nolock) {
6193             &$updatefn($vmid);
6194         } else {
6195             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6196         }
6197     } else {
6198         my $vmlist = config_list();
6199         foreach my $vmid (keys %$vmlist) {
6200             if ($nolock) {
6201                 &$updatefn($vmid);
6202             } else {
6203                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6204             }
6205         }
6206     }
6207 }
6208
6209 sub restore_proxmox_backup_archive {
6210     my ($archive, $vmid, $user, $options) = @_;
6211
6212     my $storecfg = PVE::Storage::config();
6213
6214     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6215     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6216
6217     my $fingerprint = $scfg->{fingerprint};
6218     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6219
6220     my $repo = PVE::PBSClient::get_repository($scfg);
6221
6222     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6223     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6224     local $ENV{PBS_PASSWORD} = $password;
6225     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6226
6227     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6228         PVE::Storage::parse_volname($storecfg, $archive);
6229
6230     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6231
6232     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6233
6234     my $tmpdir = "/var/tmp/vzdumptmp$$";
6235     rmtree $tmpdir;
6236     mkpath $tmpdir;
6237
6238     my $conffile = PVE::QemuConfig->config_file($vmid);
6239      # disable interrupts (always do cleanups)
6240     local $SIG{INT} =
6241         local $SIG{TERM} =
6242         local $SIG{QUIT} =
6243         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6244
6245     # Note: $oldconf is undef if VM does not exists
6246     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6247     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6248     my $new_conf_raw = '';
6249
6250     my $rpcenv = PVE::RPCEnvironment::get();
6251     my $devinfo = {};
6252
6253     eval {
6254         # enable interrupts
6255         local $SIG{INT} =
6256             local $SIG{TERM} =
6257             local $SIG{QUIT} =
6258             local $SIG{HUP} =
6259             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6260
6261         my $cfgfn = "$tmpdir/qemu-server.conf";
6262         my $firewall_config_fn = "$tmpdir/fw.conf";
6263         my $index_fn = "$tmpdir/index.json";
6264
6265         my $cmd = "restore";
6266
6267         my $param = [$pbs_backup_name, "index.json", $index_fn];
6268         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6269         my $index = PVE::Tools::file_get_contents($index_fn);
6270         $index = decode_json($index);
6271
6272         # print Dumper($index);
6273         foreach my $info (@{$index->{files}}) {
6274             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6275                 my $devname = $1;
6276                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6277                     $devinfo->{$devname}->{size} = $1;
6278                 } else {
6279                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6280                 }
6281             }
6282         }
6283
6284         my $is_qemu_server_backup = scalar(
6285             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6286         );
6287         if (!$is_qemu_server_backup) {
6288             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6289         }
6290         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6291
6292         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6293         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6294
6295         if ($has_firewall_config) {
6296             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6297             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6298
6299             my $pve_firewall_dir = '/etc/pve/firewall';
6300             mkdir $pve_firewall_dir; # make sure the dir exists
6301             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6302         }
6303
6304         my $fh = IO::File->new($cfgfn, "r") ||
6305             die "unable to read qemu-server.conf - $!\n";
6306
6307         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6308
6309         # fixme: rate limit?
6310
6311         # create empty/temp config
6312         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6313
6314         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6315
6316         # allocate volumes
6317         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6318
6319         if (!$options->{live}) {
6320             foreach my $virtdev (sort keys %$virtdev_hash) {
6321                 my $d = $virtdev_hash->{$virtdev};
6322                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6323
6324                 my $volid = $d->{volid};
6325
6326                 my $path = PVE::Storage::path($storecfg, $volid);
6327
6328                 my $pbs_restore_cmd = [
6329                     '/usr/bin/pbs-restore',
6330                     '--repository', $repo,
6331                     $pbs_backup_name,
6332                     "$d->{devname}.img.fidx",
6333                     $path,
6334                     '--verbose',
6335                     ];
6336
6337                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6338                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6339
6340                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6341                     push @$pbs_restore_cmd, '--skip-zero';
6342                 }
6343
6344                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6345                 print "restore proxmox backup image: $dbg_cmdstring\n";
6346                 run_command($pbs_restore_cmd);
6347             }
6348         }
6349
6350         $fh->seek(0, 0) || die "seek failed - $!\n";
6351
6352         my $cookie = { netcount => 0 };
6353         while (defined(my $line = <$fh>)) {
6354             $new_conf_raw .= restore_update_config_line(
6355                 $cookie,
6356                 $map,
6357                 $line,
6358                 $options->{unique},
6359             );
6360         }
6361
6362         $fh->close();
6363     };
6364     my $err = $@;
6365
6366     if ($err || !$options->{live}) {
6367         $restore_deactivate_volumes->($storecfg, $devinfo);
6368     }
6369
6370     rmtree $tmpdir;
6371
6372     if ($err) {
6373         $restore_destroy_volumes->($storecfg, $devinfo);
6374         die $err;
6375     }
6376
6377     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6378
6379     PVE::Cluster::cfs_update(); # make sure we read new file
6380
6381     eval { rescan($vmid, 1); };
6382     warn $@ if $@;
6383
6384     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6385
6386     if ($options->{live}) {
6387         eval {
6388             # enable interrupts
6389             local $SIG{INT} =
6390                 local $SIG{TERM} =
6391                 local $SIG{QUIT} =
6392                 local $SIG{HUP} =
6393                 local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6394
6395             my $conf = PVE::QemuConfig->load_config($vmid);
6396             die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6397
6398             pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6399         };
6400
6401         $err = $@;
6402         if ($err) {
6403             warn "destroying partially live-restored VM, all temporary data will be lost!\n";
6404             $restore_deactivate_volumes->($storecfg, $devinfo);
6405             $restore_destroy_volumes->($storecfg, $devinfo);
6406             PVE::QemuConfig->destroy_config($vmid);
6407             die $err;
6408         }
6409     }
6410 }
6411
6412 sub pbs_live_restore {
6413     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6414
6415     print "Starting VM for live-restore\n";
6416
6417     my $pbs_backing = {};
6418     for my $ds (keys %$restored_disks) {
6419         $ds =~ m/^drive-(.*)$/;
6420         $pbs_backing->{$1} = {
6421             repository => $repo,
6422             snapshot => $snap,
6423             archive => "$ds.img.fidx",
6424         };
6425         $pbs_backing->{$1}->{keyfile} = $keyfile if -e $keyfile;
6426     }
6427
6428     my $drives_streamed = 0;
6429     eval {
6430         # make sure HA doesn't interrupt our restore by stopping the VM
6431         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6432             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6433         }
6434
6435         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6436         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6437         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6438
6439         my $qmeventd_fd = register_qmeventd_handle($vmid);
6440
6441         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6442         # this will effectively collapse the backing image chain consisting of
6443         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6444         # removes itself once all backing images vanish with 'auto-remove=on')
6445         my $jobs = {};
6446         for my $ds (sort keys %$restored_disks) {
6447             my $job_id = "restore-$ds";
6448             mon_cmd($vmid, 'block-stream',
6449                 'job-id' => $job_id,
6450                 device => "$ds",
6451             );
6452             $jobs->{$job_id} = {};
6453         }
6454
6455         mon_cmd($vmid, 'cont');
6456         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6457
6458         print "restore-drive jobs finished successfully, removing all tracking block devices"
6459             ." to disconnect from Proxmox Backup Server\n";
6460
6461         for my $ds (sort keys %$restored_disks) {
6462             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6463         }
6464
6465         close($qmeventd_fd);
6466     };
6467
6468     my $err = $@;
6469
6470     if ($err) {
6471         warn "An error occured during live-restore: $err\n";
6472         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6473         die "live-restore failed\n";
6474     }
6475 }
6476
6477 sub restore_vma_archive {
6478     my ($archive, $vmid, $user, $opts, $comp) = @_;
6479
6480     my $readfrom = $archive;
6481
6482     my $cfg = PVE::Storage::config();
6483     my $commands = [];
6484     my $bwlimit = $opts->{bwlimit};
6485
6486     my $dbg_cmdstring = '';
6487     my $add_pipe = sub {
6488         my ($cmd) = @_;
6489         push @$commands, $cmd;
6490         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6491         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6492         $readfrom = '-';
6493     };
6494
6495     my $input = undef;
6496     if ($archive eq '-') {
6497         $input = '<&STDIN';
6498     } else {
6499         # If we use a backup from a PVE defined storage we also consider that
6500         # storage's rate limit:
6501         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6502         if (defined($volid)) {
6503             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6504             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6505             if ($readlimit) {
6506                 print STDERR "applying read rate limit: $readlimit\n";
6507                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6508                 $add_pipe->($cstream);
6509             }
6510         }
6511     }
6512
6513     if ($comp) {
6514         my $info = PVE::Storage::decompressor_info('vma', $comp);
6515         my $cmd = $info->{decompressor};
6516         push @$cmd, $readfrom;
6517         $add_pipe->($cmd);
6518     }
6519
6520     my $tmpdir = "/var/tmp/vzdumptmp$$";
6521     rmtree $tmpdir;
6522
6523     # disable interrupts (always do cleanups)
6524     local $SIG{INT} =
6525         local $SIG{TERM} =
6526         local $SIG{QUIT} =
6527         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6528
6529     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6530     POSIX::mkfifo($mapfifo, 0600);
6531     my $fifofh;
6532     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6533
6534     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6535
6536     my $oldtimeout;
6537     my $timeout = 5;
6538
6539     my $devinfo = {};
6540
6541     my $rpcenv = PVE::RPCEnvironment::get();
6542
6543     my $conffile = PVE::QemuConfig->config_file($vmid);
6544
6545     # Note: $oldconf is undef if VM does not exist
6546     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6547     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6548     my $new_conf_raw = '';
6549
6550     my %storage_limits;
6551
6552     my $print_devmap = sub {
6553         my $cfgfn = "$tmpdir/qemu-server.conf";
6554
6555         # we can read the config - that is already extracted
6556         my $fh = IO::File->new($cfgfn, "r") ||
6557             die "unable to read qemu-server.conf - $!\n";
6558
6559         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6560         if (-f $fwcfgfn) {
6561             my $pve_firewall_dir = '/etc/pve/firewall';
6562             mkdir $pve_firewall_dir; # make sure the dir exists
6563             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6564         }
6565
6566         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6567
6568         foreach my $info (values %{$virtdev_hash}) {
6569             my $storeid = $info->{storeid};
6570             next if defined($storage_limits{$storeid});
6571
6572             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6573             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6574             $storage_limits{$storeid} = $limit * 1024;
6575         }
6576
6577         foreach my $devname (keys %$devinfo) {
6578             die "found no device mapping information for device '$devname'\n"
6579                 if !$devinfo->{$devname}->{virtdev};
6580         }
6581
6582         # create empty/temp config
6583         if ($oldconf) {
6584             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6585             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6586         }
6587
6588         # allocate volumes
6589         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6590
6591         # print restore information to $fifofh
6592         foreach my $virtdev (sort keys %$virtdev_hash) {
6593             my $d = $virtdev_hash->{$virtdev};
6594             next if $d->{is_cloudinit}; # no need to restore cloudinit
6595
6596             my $storeid = $d->{storeid};
6597             my $volid = $d->{volid};
6598
6599             my $map_opts = '';
6600             if (my $limit = $storage_limits{$storeid}) {
6601                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6602             }
6603
6604             my $write_zeros = 1;
6605             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6606                 $write_zeros = 0;
6607             }
6608
6609             my $path = PVE::Storage::path($cfg, $volid);
6610
6611             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6612
6613             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6614         }
6615
6616         $fh->seek(0, 0) || die "seek failed - $!\n";
6617
6618         my $cookie = { netcount => 0 };
6619         while (defined(my $line = <$fh>)) {
6620             $new_conf_raw .= restore_update_config_line(
6621                 $cookie,
6622                 $map,
6623                 $line,
6624                 $opts->{unique},
6625             );
6626         }
6627
6628         $fh->close();
6629     };
6630
6631     eval {
6632         # enable interrupts
6633         local $SIG{INT} =
6634             local $SIG{TERM} =
6635             local $SIG{QUIT} =
6636             local $SIG{HUP} =
6637             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6638         local $SIG{ALRM} = sub { die "got timeout\n"; };
6639
6640         $oldtimeout = alarm($timeout);
6641
6642         my $parser = sub {
6643             my $line = shift;
6644
6645             print "$line\n";
6646
6647             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6648                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6649                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6650             } elsif ($line =~ m/^CTIME: /) {
6651                 # we correctly received the vma config, so we can disable
6652                 # the timeout now for disk allocation (set to 10 minutes, so
6653                 # that we always timeout if something goes wrong)
6654                 alarm(600);
6655                 &$print_devmap();
6656                 print $fifofh "done\n";
6657                 my $tmp = $oldtimeout || 0;
6658                 $oldtimeout = undef;
6659                 alarm($tmp);
6660                 close($fifofh);
6661                 $fifofh = undef;
6662             }
6663         };
6664
6665         print "restore vma archive: $dbg_cmdstring\n";
6666         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6667     };
6668     my $err = $@;
6669
6670     alarm($oldtimeout) if $oldtimeout;
6671
6672     $restore_deactivate_volumes->($cfg, $devinfo);
6673
6674     close($fifofh) if $fifofh;
6675     unlink $mapfifo;
6676     rmtree $tmpdir;
6677
6678     if ($err) {
6679         $restore_destroy_volumes->($cfg, $devinfo);
6680         die $err;
6681     }
6682
6683     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6684
6685     PVE::Cluster::cfs_update(); # make sure we read new file
6686
6687     eval { rescan($vmid, 1); };
6688     warn $@ if $@;
6689
6690     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6691 }
6692
6693 sub restore_tar_archive {
6694     my ($archive, $vmid, $user, $opts) = @_;
6695
6696     if ($archive ne '-') {
6697         my $firstfile = tar_archive_read_firstfile($archive);
6698         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6699             if $firstfile ne 'qemu-server.conf';
6700     }
6701
6702     my $storecfg = PVE::Storage::config();
6703
6704     # avoid zombie disks when restoring over an existing VM -> cleanup first
6705     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6706     # skiplock=1 because qmrestore has set the 'create' lock itself already
6707     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6708     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6709
6710     my $tocmd = "/usr/lib/qemu-server/qmextract";
6711
6712     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6713     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6714     $tocmd .= ' --prealloc' if $opts->{prealloc};
6715     $tocmd .= ' --info' if $opts->{info};
6716
6717     # tar option "xf" does not autodetect compression when read from STDIN,
6718     # so we pipe to zcat
6719     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6720         PVE::Tools::shellquote("--to-command=$tocmd");
6721
6722     my $tmpdir = "/var/tmp/vzdumptmp$$";
6723     mkpath $tmpdir;
6724
6725     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6726     local $ENV{VZDUMP_VMID} = $vmid;
6727     local $ENV{VZDUMP_USER} = $user;
6728
6729     my $conffile = PVE::QemuConfig->config_file($vmid);
6730     my $new_conf_raw = '';
6731
6732     # disable interrupts (always do cleanups)
6733     local $SIG{INT} =
6734         local $SIG{TERM} =
6735         local $SIG{QUIT} =
6736         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6737
6738     eval {
6739         # enable interrupts
6740         local $SIG{INT} =
6741             local $SIG{TERM} =
6742             local $SIG{QUIT} =
6743             local $SIG{HUP} =
6744             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6745
6746         if ($archive eq '-') {
6747             print "extracting archive from STDIN\n";
6748             run_command($cmd, input => "<&STDIN");
6749         } else {
6750             print "extracting archive '$archive'\n";
6751             run_command($cmd);
6752         }
6753
6754         return if $opts->{info};
6755
6756         # read new mapping
6757         my $map = {};
6758         my $statfile = "$tmpdir/qmrestore.stat";
6759         if (my $fd = IO::File->new($statfile, "r")) {
6760             while (defined (my $line = <$fd>)) {
6761                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6762                     $map->{$1} = $2 if $1;
6763                 } else {
6764                     print STDERR "unable to parse line in statfile - $line\n";
6765                 }
6766             }
6767             $fd->close();
6768         }
6769
6770         my $confsrc = "$tmpdir/qemu-server.conf";
6771
6772         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6773
6774         my $cookie = { netcount => 0 };
6775         while (defined (my $line = <$srcfd>)) {
6776             $new_conf_raw .= restore_update_config_line(
6777                 $cookie,
6778                 $map,
6779                 $line,
6780                 $opts->{unique},
6781             );
6782         }
6783
6784         $srcfd->close();
6785     };
6786     if (my $err = $@) {
6787         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6788         die $err;
6789     }
6790
6791     rmtree $tmpdir;
6792
6793     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6794
6795     PVE::Cluster::cfs_update(); # make sure we read new file
6796
6797     eval { rescan($vmid, 1); };
6798     warn $@ if $@;
6799 };
6800
6801 sub foreach_storage_used_by_vm {
6802     my ($conf, $func) = @_;
6803
6804     my $sidhash = {};
6805
6806     PVE::QemuConfig->foreach_volume($conf, sub {
6807         my ($ds, $drive) = @_;
6808         return if drive_is_cdrom($drive);
6809
6810         my $volid = $drive->{file};
6811
6812         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6813         $sidhash->{$sid} = $sid if $sid;
6814     });
6815
6816     foreach my $sid (sort keys %$sidhash) {
6817         &$func($sid);
6818     }
6819 }
6820
6821 my $qemu_snap_storage = {
6822     rbd => 1,
6823 };
6824 sub do_snapshots_with_qemu {
6825     my ($storecfg, $volid) = @_;
6826
6827     my $storage_name = PVE::Storage::parse_volume_id($volid);
6828     my $scfg = $storecfg->{ids}->{$storage_name};
6829     die "could not find storage '$storage_name'\n" if !defined($scfg);
6830
6831     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6832         return 1;
6833     }
6834
6835     if ($volid =~ m/\.(qcow2|qed)$/){
6836         return 1;
6837     }
6838
6839     return;
6840 }
6841
6842 sub qga_check_running {
6843     my ($vmid, $nowarn) = @_;
6844
6845     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6846     if ($@) {
6847         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6848         return 0;
6849     }
6850     return 1;
6851 }
6852
6853 sub template_create {
6854     my ($vmid, $conf, $disk) = @_;
6855
6856     my $storecfg = PVE::Storage::config();
6857
6858     PVE::QemuConfig->foreach_volume($conf, sub {
6859         my ($ds, $drive) = @_;
6860
6861         return if drive_is_cdrom($drive);
6862         return if $disk && $ds ne $disk;
6863
6864         my $volid = $drive->{file};
6865         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6866
6867         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6868         $drive->{file} = $voliddst;
6869         $conf->{$ds} = print_drive($drive);
6870         PVE::QemuConfig->write_config($vmid, $conf);
6871     });
6872 }
6873
6874 sub convert_iscsi_path {
6875     my ($path) = @_;
6876
6877     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6878         my $portal = $1;
6879         my $target = $2;
6880         my $lun = $3;
6881
6882         my $initiator_name = get_initiator_name();
6883
6884         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6885                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6886     }
6887
6888     die "cannot convert iscsi path '$path', unkown format\n";
6889 }
6890
6891 sub qemu_img_convert {
6892     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6893
6894     my $storecfg = PVE::Storage::config();
6895     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6896     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6897
6898     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6899
6900     my $cachemode;
6901     my $src_path;
6902     my $src_is_iscsi = 0;
6903     my $src_format;
6904
6905     if ($src_storeid) {
6906         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6907         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6908         $src_format = qemu_img_format($src_scfg, $src_volname);
6909         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6910         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6911         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6912     } elsif (-f $src_volid) {
6913         $src_path = $src_volid;
6914         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6915             $src_format = $1;
6916         }
6917     }
6918
6919     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6920
6921     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6922     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6923     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6924     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6925
6926     my $cmd = [];
6927     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6928     push @$cmd, '-l', "snapshot.name=$snapname"
6929         if $snapname && $src_format && $src_format eq "qcow2";
6930     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6931     push @$cmd, '-T', $cachemode if defined($cachemode);
6932
6933     if ($src_is_iscsi) {
6934         push @$cmd, '--image-opts';
6935         $src_path = convert_iscsi_path($src_path);
6936     } elsif ($src_format) {
6937         push @$cmd, '-f', $src_format;
6938     }
6939
6940     if ($dst_is_iscsi) {
6941         push @$cmd, '--target-image-opts';
6942         $dst_path = convert_iscsi_path($dst_path);
6943     } else {
6944         push @$cmd, '-O', $dst_format;
6945     }
6946
6947     push @$cmd, $src_path;
6948
6949     if (!$dst_is_iscsi && $is_zero_initialized) {
6950         push @$cmd, "zeroinit:$dst_path";
6951     } else {
6952         push @$cmd, $dst_path;
6953     }
6954
6955     my $parser = sub {
6956         my $line = shift;
6957         if($line =~ m/\((\S+)\/100\%\)/){
6958             my $percent = $1;
6959             my $transferred = int($size * $percent / 100);
6960             my $total_h = render_bytes($size, 1);
6961             my $transferred_h = render_bytes($transferred, 1);
6962
6963             print "transferred $transferred_h of $total_h ($percent%)";
6964         }
6965
6966     };
6967
6968     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6969     my $err = $@;
6970     die "copy failed: $err" if $err;
6971 }
6972
6973 sub qemu_img_format {
6974     my ($scfg, $volname) = @_;
6975
6976     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6977         return $1;
6978     } else {
6979         return "raw";
6980     }
6981 }
6982
6983 sub qemu_drive_mirror {
6984     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6985
6986     $jobs = {} if !$jobs;
6987
6988     my $qemu_target;
6989     my $format;
6990     $jobs->{"drive-$drive"} = {};
6991
6992     if ($dst_volid =~ /^nbd:/) {
6993         $qemu_target = $dst_volid;
6994         $format = "nbd";
6995     } else {
6996         my $storecfg = PVE::Storage::config();
6997         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6998
6999         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7000
7001         $format = qemu_img_format($dst_scfg, $dst_volname);
7002
7003         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7004
7005         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7006     }
7007
7008     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7009     $opts->{format} = $format if $format;
7010
7011     if (defined($src_bitmap)) {
7012         $opts->{sync} = 'incremental';
7013         $opts->{bitmap} = $src_bitmap;
7014         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7015     }
7016
7017     if (defined($bwlimit)) {
7018         $opts->{speed} = $bwlimit * 1024;
7019         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7020     } else {
7021         print "drive mirror is starting for drive-$drive\n";
7022     }
7023
7024     # if a job already runs for this device we get an error, catch it for cleanup
7025     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7026     if (my $err = $@) {
7027         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7028         warn "$@\n" if $@;
7029         die "mirroring error: $err\n";
7030     }
7031
7032     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7033 }
7034
7035 # $completion can be either
7036 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7037 # 'cancel': wait until all jobs are ready, block-job-cancel them
7038 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7039 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7040 sub qemu_drive_mirror_monitor {
7041     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7042
7043     $completion //= 'complete';
7044     $op //= "mirror";
7045
7046     eval {
7047         my $err_complete = 0;
7048
7049         my $starttime = time ();
7050         while (1) {
7051             die "block job ('$op') timed out\n" if $err_complete > 300;
7052
7053             my $stats = mon_cmd($vmid, "query-block-jobs");
7054             my $ctime = time();
7055
7056             my $running_jobs = {};
7057             for my $stat (@$stats) {
7058                 next if $stat->{type} ne $op;
7059                 $running_jobs->{$stat->{device}} = $stat;
7060             }
7061
7062             my $readycounter = 0;
7063
7064             for my $job_id (sort keys %$jobs) {
7065                 my $job = $running_jobs->{$job_id};
7066
7067                 my $vanished = !defined($job);
7068                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7069                 if($complete || ($vanished && $completion eq 'auto')) {
7070                     print "$job_id: $op-job finished\n";
7071                     delete $jobs->{$job_id};
7072                     next;
7073                 }
7074
7075                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7076
7077                 my $busy = $job->{busy};
7078                 my $ready = $job->{ready};
7079                 if (my $total = $job->{len}) {
7080                     my $transferred = $job->{offset} || 0;
7081                     my $remaining = $total - $transferred;
7082                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7083
7084                     my $duration = $ctime - $starttime;
7085                     my $total_h = render_bytes($total, 1);
7086                     my $transferred_h = render_bytes($transferred, 1);
7087
7088                     my $status = sprintf(
7089                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7090                         render_duration($duration),
7091                     );
7092
7093                     if ($ready) {
7094                         if ($busy) {
7095                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7096                         } else {
7097                             $status .= ", ready";
7098                         }
7099                     }
7100                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7101                     $jobs->{$job_id}->{ready} = $ready;
7102                 }
7103
7104                 $readycounter++ if $job->{ready};
7105             }
7106
7107             last if scalar(keys %$jobs) == 0;
7108
7109             if ($readycounter == scalar(keys %$jobs)) {
7110                 print "all '$op' jobs are ready\n";
7111
7112                 # do the complete later (or has already been done)
7113                 last if $completion eq 'skip' || $completion eq 'auto';
7114
7115                 if ($vmiddst && $vmiddst != $vmid) {
7116                     my $agent_running = $qga && qga_check_running($vmid);
7117                     if ($agent_running) {
7118                         print "freeze filesystem\n";
7119                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7120                     } else {
7121                         print "suspend vm\n";
7122                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7123                     }
7124
7125                     # if we clone a disk for a new target vm, we don't switch the disk
7126                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7127
7128                     if ($agent_running) {
7129                         print "unfreeze filesystem\n";
7130                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7131                     } else {
7132                         print "resume vm\n";
7133                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7134                     }
7135
7136                     last;
7137                 } else {
7138
7139                     for my $job_id (sort keys %$jobs) {
7140                         # try to switch the disk if source and destination are on the same guest
7141                         print "$job_id: Completing block job_id...\n";
7142
7143                         my $op;
7144                         if ($completion eq 'complete') {
7145                             $op = 'block-job-complete';
7146                         } elsif ($completion eq 'cancel') {
7147                             $op = 'block-job-cancel';
7148                         } else {
7149                             die "invalid completion value: $completion\n";
7150                         }
7151                         eval { mon_cmd($vmid, $op, device => $job_id) };
7152                         if ($@ =~ m/cannot be completed/) {
7153                             print "$job_id: block job cannot be completed, trying again.\n";
7154                             $err_complete++;
7155                         }else {
7156                             print "$job_id: Completed successfully.\n";
7157                             $jobs->{$job_id}->{complete} = 1;
7158                         }
7159                     }
7160                 }
7161             }
7162             sleep 1;
7163         }
7164     };
7165     my $err = $@;
7166
7167     if ($err) {
7168         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7169         die "block job ($op) error: $err";
7170     }
7171 }
7172
7173 sub qemu_blockjobs_cancel {
7174     my ($vmid, $jobs) = @_;
7175
7176     foreach my $job (keys %$jobs) {
7177         print "$job: Cancelling block job\n";
7178         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7179         $jobs->{$job}->{cancel} = 1;
7180     }
7181
7182     while (1) {
7183         my $stats = mon_cmd($vmid, "query-block-jobs");
7184
7185         my $running_jobs = {};
7186         foreach my $stat (@$stats) {
7187             $running_jobs->{$stat->{device}} = $stat;
7188         }
7189
7190         foreach my $job (keys %$jobs) {
7191
7192             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7193                 print "$job: Done.\n";
7194                 delete $jobs->{$job};
7195             }
7196         }
7197
7198         last if scalar(keys %$jobs) == 0;
7199
7200         sleep 1;
7201     }
7202 }
7203
7204 sub clone_disk {
7205     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7206         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7207
7208     my $newvolid;
7209
7210     if (!$full) {
7211         print "create linked clone of drive $drivename ($drive->{file})\n";
7212         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7213         push @$newvollist, $newvolid;
7214     } else {
7215
7216         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7217         $storeid = $storage if $storage;
7218
7219         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7220
7221         print "create full clone of drive $drivename ($drive->{file})\n";
7222         my $name = undef;
7223         my $size = undef;
7224         if (drive_is_cloudinit($drive)) {
7225             $name = "vm-$newvmid-cloudinit";
7226             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7227             if ($scfg->{path}) {
7228                 $name .= ".$dst_format";
7229             }
7230             $snapname = undef;
7231             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7232         } elsif ($drivename eq 'efidisk0') {
7233             $size = get_efivars_size($conf);
7234         } else {
7235             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7236         }
7237         $newvolid = PVE::Storage::vdisk_alloc(
7238             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7239         );
7240         push @$newvollist, $newvolid;
7241
7242         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7243
7244         if (drive_is_cloudinit($drive)) {
7245             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7246             # if this is the case, we have to complete any block-jobs still there from
7247             # previous drive-mirrors
7248             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7249                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7250             }
7251             goto no_data_clone;
7252         }
7253
7254         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7255         if (!$running || $snapname) {
7256             # TODO: handle bwlimits
7257             if ($drivename eq 'efidisk0') {
7258                 # the relevant data on the efidisk may be smaller than the source
7259                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7260                 # that is given by the OVMF_VARS.fd
7261                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7262                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7263
7264                 # better for Ceph if block size is not too small, see bug #3324
7265                 my $bs = 1024*1024;
7266
7267                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7268                     "if=$src_path", "of=$dst_path"]);
7269             } else {
7270                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7271             }
7272         } else {
7273
7274             my $kvmver = get_running_qemu_version ($vmid);
7275             if (!min_version($kvmver, 2, 7)) {
7276                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7277                     if $drive->{iothread};
7278             }
7279
7280             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7281                 $completion, $qga, $bwlimit);
7282         }
7283     }
7284
7285 no_data_clone:
7286     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7287
7288     my $disk = $drive;
7289     $disk->{format} = undef;
7290     $disk->{file} = $newvolid;
7291     $disk->{size} = $size if defined($size);
7292
7293     return $disk;
7294 }
7295
7296 sub get_running_qemu_version {
7297     my ($vmid) = @_;
7298     my $res = mon_cmd($vmid, "query-version");
7299     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7300 }
7301
7302 sub qemu_use_old_bios_files {
7303     my ($machine_type) = @_;
7304
7305     return if !$machine_type;
7306
7307     my $use_old_bios_files = undef;
7308
7309     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7310         $machine_type = $1;
7311         $use_old_bios_files = 1;
7312     } else {
7313         my $version = extract_version($machine_type, kvm_user_version());
7314         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7315         # load new efi bios files on migration. So this hack is required to allow
7316         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7317         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7318         $use_old_bios_files = !min_version($version, 2, 4);
7319     }
7320
7321     return ($use_old_bios_files, $machine_type);
7322 }
7323
7324 sub get_efivars_size {
7325     my ($conf) = @_;
7326     my $arch = get_vm_arch($conf);
7327     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7328     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7329     return -s $ovmf_vars;
7330 }
7331
7332 sub update_efidisk_size {
7333     my ($conf) = @_;
7334
7335     return if !defined($conf->{efidisk0});
7336
7337     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7338     $disk->{size} = get_efivars_size($conf);
7339     $conf->{efidisk0} = print_drive($disk);
7340
7341     return;
7342 }
7343
7344 sub create_efidisk($$$$$) {
7345     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7346
7347     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7348     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7349
7350     my $vars_size_b = -s $ovmf_vars;
7351     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7352     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7353     PVE::Storage::activate_volumes($storecfg, [$volid]);
7354
7355     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7356     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7357
7358     return ($volid, $size/1024);
7359 }
7360
7361 sub vm_iothreads_list {
7362     my ($vmid) = @_;
7363
7364     my $res = mon_cmd($vmid, 'query-iothreads');
7365
7366     my $iothreads = {};
7367     foreach my $iothread (@$res) {
7368         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7369     }
7370
7371     return $iothreads;
7372 }
7373
7374 sub scsihw_infos {
7375     my ($conf, $drive) = @_;
7376
7377     my $maxdev = 0;
7378
7379     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7380         $maxdev = 7;
7381     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7382         $maxdev = 1;
7383     } else {
7384         $maxdev = 256;
7385     }
7386
7387     my $controller = int($drive->{index} / $maxdev);
7388     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7389         ? "virtioscsi"
7390         : "scsihw";
7391
7392     return ($maxdev, $controller, $controller_prefix);
7393 }
7394
7395 sub windows_version {
7396     my ($ostype) = @_;
7397
7398     return 0 if !$ostype;
7399
7400     my $winversion = 0;
7401
7402     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7403         $winversion = 5;
7404     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7405         $winversion = 6;
7406     } elsif ($ostype =~ m/^win(\d+)$/) {
7407         $winversion = $1;
7408     }
7409
7410     return $winversion;
7411 }
7412
7413 sub resolve_dst_disk_format {
7414         my ($storecfg, $storeid, $src_volname, $format) = @_;
7415         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7416
7417         if (!$format) {
7418             # if no target format is specified, use the source disk format as hint
7419             if ($src_volname) {
7420                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7421                 $format = qemu_img_format($scfg, $src_volname);
7422             } else {
7423                 return $defFormat;
7424             }
7425         }
7426
7427         # test if requested format is supported - else use default
7428         my $supported = grep { $_ eq $format } @$validFormats;
7429         $format = $defFormat if !$supported;
7430         return $format;
7431 }
7432
7433 # NOTE: if this logic changes, please update docs & possibly gui logic
7434 sub find_vmstate_storage {
7435     my ($conf, $storecfg) = @_;
7436
7437     # first, return storage from conf if set
7438     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7439
7440     my ($target, $shared, $local);
7441
7442     foreach_storage_used_by_vm($conf, sub {
7443         my ($sid) = @_;
7444         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7445         my $dst = $scfg->{shared} ? \$shared : \$local;
7446         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7447     });
7448
7449     # second, use shared storage where VM has at least one disk
7450     # third, use local storage where VM has at least one disk
7451     # fall back to local storage
7452     $target = $shared // $local // 'local';
7453
7454     return $target;
7455 }
7456
7457 sub generate_uuid {
7458     my ($uuid, $uuid_str);
7459     UUID::generate($uuid);
7460     UUID::unparse($uuid, $uuid_str);
7461     return $uuid_str;
7462 }
7463
7464 sub generate_smbios1_uuid {
7465     return "uuid=".generate_uuid();
7466 }
7467
7468 sub nbd_stop {
7469     my ($vmid) = @_;
7470
7471     mon_cmd($vmid, 'nbd-server-stop');
7472 }
7473
7474 sub create_reboot_request {
7475     my ($vmid) = @_;
7476     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7477         or die "failed to create reboot trigger file: $!\n";
7478     close($fh);
7479 }
7480
7481 sub clear_reboot_request {
7482     my ($vmid) = @_;
7483     my $path = "/run/qemu-server/$vmid.reboot";
7484     my $res = 0;
7485
7486     $res = unlink($path);
7487     die "could not remove reboot request for $vmid: $!"
7488         if !$res && $! != POSIX::ENOENT;
7489
7490     return $res;
7491 }
7492
7493 sub bootorder_from_legacy {
7494     my ($conf, $bootcfg) = @_;
7495
7496     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7497     my $bootindex_hash = {};
7498     my $i = 1;
7499     foreach my $o (split(//, $boot)) {
7500         $bootindex_hash->{$o} = $i*100;
7501         $i++;
7502     }
7503
7504     my $bootorder = {};
7505
7506     PVE::QemuConfig->foreach_volume($conf, sub {
7507         my ($ds, $drive) = @_;
7508
7509         if (drive_is_cdrom ($drive, 1)) {
7510             if ($bootindex_hash->{d}) {
7511                 $bootorder->{$ds} = $bootindex_hash->{d};
7512                 $bootindex_hash->{d} += 1;
7513             }
7514         } elsif ($bootindex_hash->{c}) {
7515             $bootorder->{$ds} = $bootindex_hash->{c}
7516                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7517             $bootindex_hash->{c} += 1;
7518         }
7519     });
7520
7521     if ($bootindex_hash->{n}) {
7522         for (my $i = 0; $i < $MAX_NETS; $i++) {
7523             my $netname = "net$i";
7524             next if !$conf->{$netname};
7525             $bootorder->{$netname} = $bootindex_hash->{n};
7526             $bootindex_hash->{n} += 1;
7527         }
7528     }
7529
7530     return $bootorder;
7531 }
7532
7533 # Generate default device list for 'boot: order=' property. Matches legacy
7534 # default boot order, but with explicit device names. This is important, since
7535 # the fallback for when neither 'order' nor the old format is specified relies
7536 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7537 sub get_default_bootdevices {
7538     my ($conf) = @_;
7539
7540     my @ret = ();
7541
7542     # harddisk
7543     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7544     push @ret, $first if $first;
7545
7546     # cdrom
7547     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7548     push @ret, $first if $first;
7549
7550     # network
7551     for (my $i = 0; $i < $MAX_NETS; $i++) {
7552         my $netname = "net$i";
7553         next if !$conf->{$netname};
7554         push @ret, $netname;
7555         last;
7556     }
7557
7558     return \@ret;
7559 }
7560
7561 sub device_bootorder {
7562     my ($conf) = @_;
7563
7564     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7565
7566     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7567
7568     my $bootorder = {};
7569     if (!defined($boot) || $boot->{legacy}) {
7570         $bootorder = bootorder_from_legacy($conf, $boot);
7571     } elsif ($boot->{order}) {
7572         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7573         for my $dev (PVE::Tools::split_list($boot->{order})) {
7574             $bootorder->{$dev} = $i++;
7575         }
7576     }
7577
7578     return $bootorder;
7579 }
7580
7581 sub register_qmeventd_handle {
7582     my ($vmid) = @_;
7583
7584     my $fh;
7585     my $peer = "/var/run/qmeventd.sock";
7586     my $count = 0;
7587
7588     for (;;) {
7589         $count++;
7590         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7591         last if $fh;
7592         if ($! != EINTR && $! != EAGAIN) {
7593             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7594         }
7595         if ($count > 4) {
7596             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7597               . "after $count retries\n";
7598         }
7599         usleep(25000);
7600     }
7601
7602     # send handshake to mark VM as backing up
7603     print $fh to_json({vzdump => {vmid => "$vmid"}});
7604
7605     # return handle to be closed later when inhibit is no longer required
7606     return $fh;
7607 }
7608
7609 # bash completion helper
7610
7611 sub complete_backup_archives {
7612     my ($cmdname, $pname, $cvalue) = @_;
7613
7614     my $cfg = PVE::Storage::config();
7615
7616     my $storeid;
7617
7618     if ($cvalue =~ m/^([^:]+):/) {
7619         $storeid = $1;
7620     }
7621
7622     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7623
7624     my $res = [];
7625     foreach my $id (keys %$data) {
7626         foreach my $item (@{$data->{$id}}) {
7627             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7628             push @$res, $item->{volid} if defined($item->{volid});
7629         }
7630     }
7631
7632     return $res;
7633 }
7634
7635 my $complete_vmid_full = sub {
7636     my ($running) = @_;
7637
7638     my $idlist = vmstatus();
7639
7640     my $res = [];
7641
7642     foreach my $id (keys %$idlist) {
7643         my $d = $idlist->{$id};
7644         if (defined($running)) {
7645             next if $d->{template};
7646             next if $running && $d->{status} ne 'running';
7647             next if !$running && $d->{status} eq 'running';
7648         }
7649         push @$res, $id;
7650
7651     }
7652     return $res;
7653 };
7654
7655 sub complete_vmid {
7656     return &$complete_vmid_full();
7657 }
7658
7659 sub complete_vmid_stopped {
7660     return &$complete_vmid_full(0);
7661 }
7662
7663 sub complete_vmid_running {
7664     return &$complete_vmid_full(1);
7665 }
7666
7667 sub complete_storage {
7668
7669     my $cfg = PVE::Storage::config();
7670     my $ids = $cfg->{ids};
7671
7672     my $res = [];
7673     foreach my $sid (keys %$ids) {
7674         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7675         next if !$ids->{$sid}->{content}->{images};
7676         push @$res, $sid;
7677     }
7678
7679     return $res;
7680 }
7681
7682 sub complete_migration_storage {
7683     my ($cmd, $param, $current_value, $all_args) = @_;
7684
7685     my $targetnode = @$all_args[1];
7686
7687     my $cfg = PVE::Storage::config();
7688     my $ids = $cfg->{ids};
7689
7690     my $res = [];
7691     foreach my $sid (keys %$ids) {
7692         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7693         next if !$ids->{$sid}->{content}->{images};
7694         push @$res, $sid;
7695     }
7696
7697     return $res;
7698 }
7699
7700 sub vm_is_paused {
7701     my ($vmid) = @_;
7702     my $qmpstatus = eval {
7703         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7704         mon_cmd($vmid, "query-status");
7705     };
7706     warn "$@\n" if $@;
7707     return $qmpstatus && $qmpstatus->{status} eq "paused";
7708 }
7709
7710 1;