PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 369     },
 370     ostype => {
 371         optional => 1,
 372         type => 'string',
 373         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 374         description => "Specify guest operating system.",
 375         verbose_description => <<EODESC,
 376 Specify guest operating system. This is used to enable special
 377 optimization/features for specific operating systems:
 378
 379 [horizontal]
 380 other;; unspecified OS
 381 wxp;; Microsoft Windows XP
 382 w2k;; Microsoft Windows 2000
 383 w2k3;; Microsoft Windows 2003
 384 w2k8;; Microsoft Windows 2008
 385 wvista;; Microsoft Windows Vista
 386 win7;; Microsoft Windows 7
 387 win8;; Microsoft Windows 8/2012/2012r2
 388 win10;; Microsoft Windows 10/2016/2019
 389 l24;; Linux 2.4 Kernel
 390 l26;; Linux 2.6 - 5.X Kernel
 391 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 392 EODESC
 393     },
 394     boot => {
 395         optional => 1,
 396         type => 'string', format => 'pve-qm-boot',
 397         description => "Specify guest boot order. Use with 'order=', usage with"
 398                      . " no key or 'legacy=' is deprecated.",
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     keephugepages => {
 440         optional => 1,
 441         type => 'boolean',
 442         default => 0,
 443         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 444             ." after VM shutdown and can be used for subsequent starts.",
 445     },
 446     vcpus => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "Number of hotplugged vcpus.",
 450         minimum => 1,
 451         default => 0,
 452     },
 453     acpi => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Enable/disable ACPI.",
 457         default => 1,
 458     },
 459     agent => {
 460         optional => 1,
 461         description => "Enable/disable Qemu GuestAgent and its properties.",
 462         type => 'string',
 463         format => $agent_fmt,
 464     },
 465     kvm => {
 466         optional => 1,
 467         type => 'boolean',
 468         description => "Enable/disable KVM hardware virtualization.",
 469         default => 1,
 470     },
 471     tdf => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable time drift fix.",
 475         default => 0,
 476     },
 477     localtime => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Set the real time clock to local time. This is enabled by default if ostype"
 481             ." indicates a Microsoft OS.",
 482     },
 483     freeze => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 487     },
 488     vga => {
 489         optional => 1,
 490         type => 'string', format => $vga_fmt,
 491         description => "Configure the VGA hardware.",
 492         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 493             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 494             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 495             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 496             ." display server. For win* OS you can select how many independent displays you want,"
 497             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 498             ." using a serial device as terminal.",
 499     },
 500     watchdog => {
 501         optional => 1,
 502         type => 'string', format => 'pve-qm-watchdog',
 503         description => "Create a virtual hardware watchdog device.",
 504         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 505             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 506             ." else the watchdog will reset the guest (or execute the respective action specified)",
 507     },
 508     startdate => {
 509         optional => 1,
 510         type => 'string',
 511         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 512         description => "Set the initial date of the real time clock. Valid format for date are:"
 513             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 514         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 515         default => 'now',
 516     },
 517     startup =>  get_standard_option('pve-startup-order'),
 518     template => {
 519         optional => 1,
 520         type => 'boolean',
 521         description => "Enable/disable Template.",
 522         default => 0,
 523     },
 524     args => {
 525         optional => 1,
 526         type => 'string',
 527         description => "Arbitrary arguments passed to kvm.",
 528         verbose_description => <<EODESCR,
 529 Arbitrary arguments passed to kvm, for example:
 530
 531 args: -no-reboot -no-hpet
 532
 533 NOTE: this option is for experts only.
 534 EODESCR
 535     },
 536     tablet => {
 537         optional => 1,
 538         type => 'boolean',
 539         default => 1,
 540         description => "Enable/disable the USB tablet device.",
 541         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 542             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 543             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 544             ." may consider disabling this to save some context switches. This is turned off by"
 545             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 546     },
 547     migrate_speed => {
 548         optional => 1,
 549         type => 'integer',
 550         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 551         minimum => 0,
 552         default => 0,
 553     },
 554     migrate_downtime => {
 555         optional => 1,
 556         type => 'number',
 557         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 558         minimum => 0,
 559         default => 0.1,
 560     },
 561     cdrom => {
 562         optional => 1,
 563         type => 'string', format => 'pve-qm-ide',
 564         typetext => '<volume>',
 565         description => "This is an alias for option -ide2",
 566     },
 567     cpu => {
 568         optional => 1,
 569         description => "Emulated CPU type.",
 570         type => 'string',
 571         format => 'pve-vm-cpu-conf',
 572     },
 573     parent => get_standard_option('pve-snapshot-name', {
 574         optional => 1,
 575         description => "Parent snapshot name. This is used internally, and should not be modified.",
 576     }),
 577     snaptime => {
 578         optional => 1,
 579         description => "Timestamp for snapshots.",
 580         type => 'integer',
 581         minimum => 0,
 582     },
 583     vmstate => {
 584         optional => 1,
 585         type => 'string', format => 'pve-volume-id',
 586         description => "Reference to a volume which stores the VM state. This is used internally"
 587             ." for snapshots.",
 588     },
 589     vmstatestorage => get_standard_option('pve-storage-id', {
 590         description => "Default storage for VM state volumes/files.",
 591         optional => 1,
 592     }),
 593     runningmachine => get_standard_option('pve-qemu-machine', {
 594         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 595             ." for snapshots.",
 596     }),
 597     runningcpu => {
 598         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 599             ." internally for snapshots.",
 600         optional => 1,
 601         type => 'string',
 602         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 603         format_description => 'QEMU -cpu parameter'
 604     },
 605     machine => get_standard_option('pve-qemu-machine'),
 606     arch => {
 607         description => "Virtual processor architecture. Defaults to the host.",
 608         optional => 1,
 609         type => 'string',
 610         enum => [qw(x86_64 aarch64)],
 611     },
 612     smbios1 => {
 613         description => "Specify SMBIOS type 1 fields.",
 614         type => 'string', format => 'pve-qm-smbios1',
 615         maxLength => 512,
 616         optional => 1,
 617     },
 618     protection => {
 619         optional => 1,
 620         type => 'boolean',
 621         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 622             ." remove disk operations.",
 623         default => 0,
 624     },
 625     bios => {
 626         optional => 1,
 627         type => 'string',
 628         enum => [ qw(seabios ovmf) ],
 629         description => "Select BIOS implementation.",
 630         default => 'seabios',
 631     },
 632     vmgenid => {
 633         type => 'string',
 634         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 635         format_description => 'UUID',
 636         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 637             ." to disable explicitly.",
 638         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 639             ." value identifier to the guest OS. This allows to notify the guest operating system"
 640             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 641             ." execution or creation from a template). The guest operating system notices the"
 642             ." change, and is then able to react as appropriate by marking its copies of"
 643             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 644             ."Note that auto-creation only works when done through API/CLI create or update methods"
 645             .", but not when manually editing the config file.",
 646         default => "1 (autogenerated)",
 647         optional => 1,
 648     },
 649     hookscript => {
 650         type => 'string',
 651         format => 'pve-volume-id',
 652         optional => 1,
 653         description => "Script that will be executed during various steps in the vms lifetime.",
 654     },
 655     ivshmem => {
 656         type => 'string',
 657         format => $ivshmem_fmt,
 658         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 659             ." the host.",
 660         optional => 1,
 661     },
 662     audio0 => {
 663         type => 'string',
 664         format => $audio_fmt,
 665         description => "Configure a audio device, useful in combination with QXL/Spice.",
 666         optional => 1
 667     },
 668     spice_enhancements => {
 669         type => 'string',
 670         format => $spice_enhancements_fmt,
 671         description => "Configure additional enhancements for SPICE.",
 672         optional => 1
 673     },
 674     tags => {
 675         type => 'string', format => 'pve-tag-list',
 676         description => 'Tags of the VM. This is only meta information.',
 677         optional => 1,
 678     },
 679     rng0 => {
 680         type => 'string',
 681         format => $rng_fmt,
 682         description => "Configure a VirtIO-based Random Number Generator.",
 683         optional => 1,
 684     },
 685 };
 686
 687 my $cicustom_fmt = {
 688     meta => {
 689         type => 'string',
 690         optional => 1,
 691         description => 'Specify a custom file containing all meta data passed to the VM via"
 692             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 693         format => 'pve-volume-id',
 694         format_description => 'volume',
 695     },
 696     network => {
 697         type => 'string',
 698         optional => 1,
 699         description => 'Specify a custom file containing all network data passed to the VM via'
 700             .' cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704     user => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all user data passed to the VM via'
 708             .' cloud-init.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712 };
 713 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 714
 715 my $confdesc_cloudinit = {
 716     citype => {
 717         optional => 1,
 718         type => 'string',
 719         description => 'Specifies the cloud-init configuration format. The default depends on the'
 720             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 721             .' and `configdrive2` for windows.',
 722         enum => ['configdrive2', 'nocloud', 'opennebula'],
 723     },
 724     ciuser => {
 725         optional => 1,
 726         type => 'string',
 727         description => "cloud-init: User name to change ssh keys and password for instead of the"
 728             ." image's configured default user.",
 729     },
 730     cipassword => {
 731         optional => 1,
 732         type => 'string',
 733         description => 'cloud-init: Password to assign the user. Using this is generally not'
 734             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 735             .' support hashed passwords.',
 736     },
 737     cicustom => {
 738         optional => 1,
 739         type => 'string',
 740         description => 'cloud-init: Specify custom files to replace the automatically generated'
 741             .' ones at start.',
 742         format => 'pve-qm-cicustom',
 743     },
 744     searchdomain => {
 745         optional => 1,
 746         type => 'string',
 747         description => "cloud-init: Sets DNS search domains for a container. Create will'
 748             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 749             .' are set.",
 750     },
 751     nameserver => {
 752         optional => 1,
 753         type => 'string', format => 'address-list',
 754         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 755             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 756             .' are set.",
 757     },
 758     sshkeys => {
 759         optional => 1,
 760         type => 'string',
 761         format => 'urlencoded',
 762         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 763     },
 764 };
 765
 766 # what about other qemu settings ?
 767 #cpu => 'string',
 768 #machine => 'string',
 769 #fda => 'file',
 770 #fdb => 'file',
 771 #mtdblock => 'file',
 772 #sd => 'file',
 773 #pflash => 'file',
 774 #snapshot => 'bool',
 775 #bootp => 'file',
 776 ##tftp => 'dir',
 777 ##smb => 'dir',
 778 #kernel => 'file',
 779 #append => 'string',
 780 #initrd => 'file',
 781 ##soundhw => 'string',
 782
 783 while (my ($k, $v) = each %$confdesc) {
 784     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 785 }
 786
 787 my $MAX_USB_DEVICES = 5;
 788 my $MAX_NETS = 32;
 789 my $MAX_SERIAL_PORTS = 4;
 790 my $MAX_PARALLEL_PORTS = 3;
 791 my $MAX_NUMA = 8;
 792
 793 my $numa_fmt = {
 794     cpus => {
 795         type => "string",
 796         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 797         description => "CPUs accessing this NUMA node.",
 798         format_description => "id[-id];...",
 799     },
 800     memory => {
 801         type => "number",
 802         description => "Amount of memory this NUMA node provides.",
 803         optional => 1,
 804     },
 805     hostnodes => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "Host NUMA nodes to use.",
 809         format_description => "id[-id];...",
 810         optional => 1,
 811     },
 812     policy => {
 813         type => 'string',
 814         enum => [qw(preferred bind interleave)],
 815         description => "NUMA allocation policy.",
 816         optional => 1,
 817     },
 818 };
 819 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 820 my $numadesc = {
 821     optional => 1,
 822     type => 'string', format => $numa_fmt,
 823     description => "NUMA topology.",
 824 };
 825 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 826
 827 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 828     $confdesc->{"numa$i"} = $numadesc;
 829 }
 830
 831 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 832                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 833                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 834 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 835
 836 my $net_fmt_bridge_descr = <<__EOD__;
 837 Bridge to attach the network device to. The Proxmox VE standard bridge
 838 is called 'vmbr0'.
 839
 840 If you do not specify a bridge, we create a kvm user (NATed) network
 841 device, which provides DHCP and DNS services. The following addresses
 842 are used:
 843
 844  10.0.2.2   Gateway
 845  10.0.2.3   DNS Server
 846  10.0.2.4   SMB Server
 847
 848 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 849 __EOD__
 850
 851 my $net_fmt = {
 852     macaddr  => get_standard_option('mac-addr', {
 853         description => "MAC address. That address must be unique withing your network. This is"
 854             ." automatically generated if not specified.",
 855     }),
 856     model => {
 857         type => 'string',
 858         description => "Network Card Model. The 'virtio' model provides the best performance with"
 859             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 860             ." best to use 'e1000'.",
 861         enum => $nic_model_list,
 862         default_key => 1,
 863     },
 864     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 865     bridge => {
 866         type => 'string',
 867         description => $net_fmt_bridge_descr,
 868         format_description => 'bridge',
 869         pattern => '[-_.\w\d]+',
 870         optional => 1,
 871     },
 872     queues => {
 873         type => 'integer',
 874         minimum => 0, maximum => 16,
 875         description => 'Number of packet queues to be used on the device.',
 876         optional => 1,
 877     },
 878     rate => {
 879         type => 'number',
 880         minimum => 0,
 881         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 882         optional => 1,
 883     },
 884     tag => {
 885         type => 'integer',
 886         minimum => 1, maximum => 4094,
 887         description => 'VLAN tag to apply to packets on this interface.',
 888         optional => 1,
 889     },
 890     trunks => {
 891         type => 'string',
 892         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 893         description => 'VLAN trunks to pass through this interface.',
 894         format_description => 'vlanid[;vlanid...]',
 895         optional => 1,
 896     },
 897     firewall => {
 898         type => 'boolean',
 899         description => 'Whether this interface should be protected by the firewall.',
 900         optional => 1,
 901     },
 902     link_down => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be disconnected (like pulling the plug).',
 905         optional => 1,
 906     },
 907     mtu => {
 908         type => 'integer',
 909         minimum => 1, maximum => 65520,
 910         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 911         optional => 1,
 912     },
 913 };
 914
 915 my $netdesc = {
 916     optional => 1,
 917     type => 'string', format => $net_fmt,
 918     description => "Specify network devices.",
 919 };
 920
 921 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 922
 923 my $ipconfig_fmt = {
 924     ip => {
 925         type => 'string',
 926         format => 'pve-ipv4-config',
 927         format_description => 'IPv4Format/CIDR',
 928         description => 'IPv4 address in CIDR format.',
 929         optional => 1,
 930         default => 'dhcp',
 931     },
 932     gw => {
 933         type => 'string',
 934         format => 'ipv4',
 935         format_description => 'GatewayIPv4',
 936         description => 'Default gateway for IPv4 traffic.',
 937         optional => 1,
 938         requires => 'ip',
 939     },
 940     ip6 => {
 941         type => 'string',
 942         format => 'pve-ipv6-config',
 943         format_description => 'IPv6Format/CIDR',
 944         description => 'IPv6 address in CIDR format.',
 945         optional => 1,
 946         default => 'dhcp',
 947     },
 948     gw6 => {
 949         type => 'string',
 950         format => 'ipv6',
 951         format_description => 'GatewayIPv6',
 952         description => 'Default gateway for IPv6 traffic.',
 953         optional => 1,
 954         requires => 'ip6',
 955     },
 956 };
 957 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 958 my $ipconfigdesc = {
 959     optional => 1,
 960     type => 'string', format => 'pve-qm-ipconfig',
 961     description => <<'EODESCR',
 962 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 963
 964 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 965
 966 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 967 gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 969 cloud-init 19.4 or newer.
 970
 971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 972 dhcp on IPv4.
 973 EODESCR
 974 };
 975 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 976
 977 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 978     $confdesc->{"net$i"} = $netdesc;
 979     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 980 }
 981
 982 foreach my $key (keys %$confdesc_cloudinit) {
 983     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 984 }
 985
 986 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 987 sub verify_volume_id_or_qm_path {
 988     my ($volid, $noerr) = @_;
 989
 990     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 991         return $volid;
 992     }
 993
 994     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 995     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 996     if ($@) {
 997         return if $noerr;
 998         die $@;
 999     }
1000     return $volid;
1001 }
1002
1003 my $usb_fmt = {
1004     host => {
1005         default_key => 1,
1006         type => 'string', format => 'pve-qm-usb-device',
1007         format_description => 'HOSTUSBDEVICE|spice',
1008         description => <<EODESCR,
1009 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1010
1011  'bus-port(.port)*' (decimal numbers) or
1012  'vendor_id:product_id' (hexadeciaml numbers) or
1013  'spice'
1014
1015 You can use the 'lsusb -t' command to list existing usb devices.
1016
1017 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1018 machines - use with special care.
1019
1020 The value 'spice' can be used to add a usb redirection devices for spice.
1021 EODESCR
1022     },
1023     usb3 => {
1024         optional => 1,
1025         type => 'boolean',
1026         description => "Specifies whether if given host option is a USB3 device or port.",
1027         default => 0,
1028     },
1029 };
1030
1031 my $usbdesc = {
1032     optional => 1,
1033     type => 'string', format => $usb_fmt,
1034     description => "Configure an USB device (n is 0 to 4).",
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1037
1038 my $serialdesc = {
1039         optional => 1,
1040         type => 'string',
1041         pattern => '(/dev/.+|socket)',
1042         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1043         verbose_description =>  <<EODESCR,
1044 Create a serial device inside the VM (n is 0 to 3), and pass through a
1045 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046 host side (use 'qm terminal' to open a terminal connection).
1047
1048 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1049 use with special care.
1050
1051 CAUTION: Experimental! User reported problems with this option.
1052 EODESCR
1053 };
1054
1055 my $paralleldesc= {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1059         description =>  "Map host parallel devices (n is 0 to 2).",
1060         verbose_description =>  <<EODESCR,
1061 Map host parallel devices (n is 0 to 2).
1062
1063 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1064 machines - use with special care.
1065
1066 CAUTION: Experimental! User reported problems with this option.
1067 EODESCR
1068 };
1069
1070 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1071     $confdesc->{"parallel$i"} = $paralleldesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1075     $confdesc->{"serial$i"} = $serialdesc;
1076 }
1077
1078 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1079     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1080 }
1081
1082 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1083     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1084 }
1085
1086 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1087     $confdesc->{"usb$i"} = $usbdesc;
1088 }
1089
1090 my $boot_fmt = {
1091     legacy => {
1092         optional => 1,
1093         default_key => 1,
1094         type => 'string',
1095         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1096                      . " Deprecated, use 'order=' instead.",
1097         pattern => '[acdn]{1,4}',
1098         format_description => "[acdn]{1,4}",
1099
1100         # note: this is also the fallback if boot: is not given at all
1101         default => 'cdn',
1102     },
1103     order => {
1104         optional => 1,
1105         type => 'string',
1106         format => 'pve-qm-bootdev-list',
1107         format_description => "device[;device...]",
1108         description => <<EODESC,
1109 The guest will attempt to boot from devices in the order they appear here.
1110
1111 Disks, optical drives and passed-through storage USB devices will be directly
1112 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1113 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1114
1115 Note that only devices in this list will be marked as bootable and thus loaded
1116 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1117 (e.g. software-raid), you need to specify all of them here.
1118
1119 Overrides the deprecated 'legacy=[acdn]*' value when given.
1120 EODESC
1121     },
1122 };
1123 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1124
1125 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1126 sub verify_bootdev {
1127     my ($dev, $noerr) = @_;
1128
1129     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1130
1131     my $check = sub {
1132         my ($base) = @_;
1133         return 0 if $dev !~ m/^$base\d+$/;
1134         return 0 if !$confdesc->{$dev};
1135         return 1;
1136     };
1137
1138     return $dev if $check->("net");
1139     return $dev if $check->("usb");
1140     return $dev if $check->("hostpci");
1141
1142     return if $noerr;
1143     die "invalid boot device '$dev'\n";
1144 }
1145
1146 sub print_bootorder {
1147     my ($devs) = @_;
1148     return "" if !@$devs;
1149     my $data = { order => join(';', @$devs) };
1150     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1151 }
1152
1153 my $kvm_api_version = 0;
1154
1155 sub kvm_version {
1156     return $kvm_api_version if $kvm_api_version;
1157
1158     open my $fh, '<', '/dev/kvm' or return;
1159
1160     # 0xae00 => KVM_GET_API_VERSION
1161     $kvm_api_version = ioctl($fh, 0xae00, 0);
1162     close($fh);
1163
1164     return $kvm_api_version;
1165 }
1166
1167 my $kvm_user_version = {};
1168 my $kvm_mtime = {};
1169
1170 sub kvm_user_version {
1171     my ($binary) = @_;
1172
1173     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1174     my $st = stat($binary);
1175
1176     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1177     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1178         $cachedmtime == $st->mtime;
1179
1180     $kvm_user_version->{$binary} = 'unknown';
1181     $kvm_mtime->{$binary} = $st->mtime;
1182
1183     my $code = sub {
1184         my $line = shift;
1185         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186             $kvm_user_version->{$binary} = $2;
1187         }
1188     };
1189
1190     eval { run_command([$binary, '--version'], outfunc => $code); };
1191     warn $@ if $@;
1192
1193     return $kvm_user_version->{$binary};
1194
1195 }
1196 my sub extract_version {
1197     my ($machine_type, $version) = @_;
1198     $version = kvm_user_version() if !defined($version);
1199     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1200 }
1201
1202 sub kernel_has_vhost_net {
1203     return -c '/dev/vhost-net';
1204 }
1205
1206 sub option_exists {
1207     my $key = shift;
1208     return defined($confdesc->{$key});
1209 }
1210
1211 my $cdrom_path;
1212 sub get_cdrom_path {
1213
1214     return  $cdrom_path if $cdrom_path;
1215
1216     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1217     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1218     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1219 }
1220
1221 sub get_iso_path {
1222     my ($storecfg, $vmid, $cdrom) = @_;
1223
1224     if ($cdrom eq 'cdrom') {
1225         return get_cdrom_path();
1226     } elsif ($cdrom eq 'none') {
1227         return '';
1228     } elsif ($cdrom =~ m|^/|) {
1229         return $cdrom;
1230     } else {
1231         return PVE::Storage::path($storecfg, $cdrom);
1232     }
1233 }
1234
1235 # try to convert old style file names to volume IDs
1236 sub filename_to_volume_id {
1237     my ($vmid, $file, $media) = @_;
1238
1239      if (!($file eq 'none' || $file eq 'cdrom' ||
1240           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1241
1242         return if $file =~ m|/|;
1243
1244         if ($media && $media eq 'cdrom') {
1245             $file = "local:iso/$file";
1246         } else {
1247             $file = "local:$vmid/$file";
1248         }
1249     }
1250
1251     return $file;
1252 }
1253
1254 sub verify_media_type {
1255     my ($opt, $vtype, $media) = @_;
1256
1257     return if !$media;
1258
1259     my $etype;
1260     if ($media eq 'disk') {
1261         $etype = 'images';
1262     } elsif ($media eq 'cdrom') {
1263         $etype = 'iso';
1264     } else {
1265         die "internal error";
1266     }
1267
1268     return if ($vtype eq $etype);
1269
1270     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1271 }
1272
1273 sub cleanup_drive_path {
1274     my ($opt, $storecfg, $drive) = @_;
1275
1276     # try to convert filesystem paths to volume IDs
1277
1278     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1279         ($drive->{file} !~ m|^/dev/.+|) &&
1280         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1281         ($drive->{file} !~ m/^\d+$/)) {
1282         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1283         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1284             if !$vtype;
1285         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1286         verify_media_type($opt, $vtype, $drive->{media});
1287         $drive->{file} = $volid;
1288     }
1289
1290     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1291 }
1292
1293 sub parse_hotplug_features {
1294     my ($data) = @_;
1295
1296     my $res = {};
1297
1298     return $res if $data eq '0';
1299
1300     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1301
1302     foreach my $feature (PVE::Tools::split_list($data)) {
1303         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304             $res->{$1} = 1;
1305         } else {
1306             die "invalid hotplug feature '$feature'\n";
1307         }
1308     }
1309     return $res;
1310 }
1311
1312 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1313 sub pve_verify_hotplug_features {
1314     my ($value, $noerr) = @_;
1315
1316     return $value if parse_hotplug_features($value);
1317
1318     return if $noerr;
1319
1320     die "unable to parse hotplug option\n";
1321 }
1322
1323 sub scsi_inquiry {
1324     my($fh, $noerr) = @_;
1325
1326     my $SG_IO = 0x2285;
1327     my $SG_GET_VERSION_NUM = 0x2282;
1328
1329     my $versionbuf = "\x00" x 8;
1330     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1331     if (!$ret) {
1332         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333         return;
1334     }
1335     my $version = unpack("I", $versionbuf);
1336     if ($version < 30000) {
1337         die "scsi generic interface too old\n"  if !$noerr;
1338         return;
1339     }
1340
1341     my $buf = "\x00" x 36;
1342     my $sensebuf = "\x00" x 8;
1343     my $cmd = pack("C x3 C x1", 0x12, 36);
1344
1345     # see /usr/include/scsi/sg.h
1346     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1347
1348     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1349                       length($sensebuf), 0, length($buf), $buf,
1350                       $cmd, $sensebuf, 6000);
1351
1352     $ret = ioctl($fh, $SG_IO, $packet);
1353     if (!$ret) {
1354         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1355         return;
1356     }
1357
1358     my @res = unpack($sg_io_hdr_t, $packet);
1359     if ($res[17] || $res[18]) {
1360         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1361         return;
1362     }
1363
1364     my $res = {};
1365     (my $byte0, my $byte1, $res->{vendor},
1366      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1367
1368     $res->{removable} = $byte1 & 128 ? 1 : 0;
1369     $res->{type} = $byte0 & 31;
1370
1371     return $res;
1372 }
1373
1374 sub path_is_scsi {
1375     my ($path) = @_;
1376
1377     my $fh = IO::File->new("+<$path") || return;
1378     my $res = scsi_inquiry($fh, 1);
1379     close($fh);
1380
1381     return $res;
1382 }
1383
1384 sub print_tabletdevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1388
1389     # we use uhci for old VMs because tablet driver was buggy in older qemu
1390     my $usbbus;
1391     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1392         $usbbus = 'ehci';
1393     } else {
1394         $usbbus = 'uhci';
1395     }
1396
1397     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 }
1399
1400 sub print_keyboarddevice_full {
1401     my ($conf, $arch, $machine) = @_;
1402
1403     return if $arch ne 'aarch64';
1404
1405     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 }
1407
1408 my sub get_drive_id {
1409     my ($drive) = @_;
1410     return "$drive->{interface}$drive->{index}";
1411 }
1412
1413 sub print_drivedevice_full {
1414     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1415
1416     my $device = '';
1417     my $maxdev = 0;
1418
1419     my $drive_id = get_drive_id($drive);
1420     if ($drive->{interface} eq 'virtio') {
1421         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1422         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1423         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1424     } elsif ($drive->{interface} eq 'scsi') {
1425
1426         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1427         my $unit = $drive->{index} % $maxdev;
1428         my $devicetype = 'hd';
1429         my $path = '';
1430         if (drive_is_cdrom($drive)) {
1431             $devicetype = 'cd';
1432         } else {
1433             if ($drive->{file} =~ m|^/|) {
1434                 $path = $drive->{file};
1435                 if (my $info = path_is_scsi($path)) {
1436                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1437                         $devicetype = 'block';
1438                     } elsif ($info->{type} == 1) { # tape
1439                         $devicetype = 'generic';
1440                     }
1441                 }
1442             } else {
1443                  $path = PVE::Storage::path($storecfg, $drive->{file});
1444             }
1445
1446             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1447             my $version = extract_version($machine_type, kvm_user_version());
1448             if ($path =~ m/^iscsi\:\/\// &&
1449                !min_version($version, 4, 1)) {
1450                 $devicetype = 'generic';
1451             }
1452         }
1453
1454         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1455             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1456         } else {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1458                 .",lun=$drive->{index}";
1459         }
1460         $device .= ",drive=drive-$drive_id,id=$drive_id";
1461
1462         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1463             $device .= ",rotation_rate=1";
1464         }
1465         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1466
1467     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1468         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1469         my $controller = int($drive->{index} / $maxdev);
1470         my $unit = $drive->{index} % $maxdev;
1471         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1472
1473         $device = "ide-$devicetype";
1474         if ($drive->{interface} eq 'ide') {
1475             $device .= ",bus=ide.$controller,unit=$unit";
1476         } else {
1477             $device .= ",bus=ahci$controller.$unit";
1478         }
1479         $device .= ",drive=drive-$drive_id,id=$drive_id";
1480
1481         if ($devicetype eq 'hd') {
1482             if (my $model = $drive->{model}) {
1483                 $model = URI::Escape::uri_unescape($model);
1484                 $device .= ",model=$model";
1485             }
1486             if ($drive->{ssd}) {
1487                 $device .= ",rotation_rate=1";
1488             }
1489         }
1490         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1491     } elsif ($drive->{interface} eq 'usb') {
1492         die "implement me";
1493         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1494     } else {
1495         die "unsupported interface type";
1496     }
1497
1498     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1499
1500     if (my $serial = $drive->{serial}) {
1501         $serial = URI::Escape::uri_unescape($serial);
1502         $device .= ",serial=$serial";
1503     }
1504
1505
1506     return $device;
1507 }
1508
1509 sub get_initiator_name {
1510     my $initiator;
1511
1512     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1513     while (defined(my $line = <$fh>)) {
1514         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1515         $initiator = $1;
1516         last;
1517     }
1518     $fh->close();
1519
1520     return $initiator;
1521 }
1522
1523 sub print_drive_commandline_full {
1524     my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1525
1526     my $path;
1527     my $volid = $drive->{file};
1528     my $format = $drive->{format};
1529     my $drive_id = get_drive_id($drive);
1530
1531     if (drive_is_cdrom($drive)) {
1532         $path = get_iso_path($storecfg, $vmid, $volid);
1533         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1534     } else {
1535         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1536         if ($storeid) {
1537             $path = PVE::Storage::path($storecfg, $volid);
1538             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1539             $format //= qemu_img_format($scfg, $volname);
1540         } else {
1541             $path = $volid;
1542             $format //= "raw";
1543         }
1544    }
1545
1546    my $is_rbd = $path =~ m/^rbd:/;
1547
1548     my $opts = '';
1549     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1550     foreach my $o (@qemu_drive_options) {
1551         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1552     }
1553
1554     # snapshot only accepts on|off
1555     if (defined($drive->{snapshot})) {
1556         my $v = $drive->{snapshot} ? 'on' : 'off';
1557         $opts .= ",snapshot=$v";
1558     }
1559
1560     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1561         my ($dir, $qmpname) = @$type;
1562         if (my $v = $drive->{"mbps$dir"}) {
1563             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1564         }
1565         if (my $v = $drive->{"mbps${dir}_max"}) {
1566             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1567         }
1568         if (my $v = $drive->{"bps${dir}_max_length"}) {
1569             $opts .= ",throttling.bps$qmpname-max-length=$v";
1570         }
1571         if (my $v = $drive->{"iops${dir}"}) {
1572             $opts .= ",throttling.iops$qmpname=$v";
1573         }
1574         if (my $v = $drive->{"iops${dir}_max"}) {
1575             $opts .= ",throttling.iops$qmpname-max=$v";
1576         }
1577         if (my $v = $drive->{"iops${dir}_max_length"}) {
1578             $opts .= ",throttling.iops$qmpname-max-length=$v";
1579         }
1580     }
1581
1582     if ($pbs_name) {
1583         $format = "rbd" if $is_rbd;
1584         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1585             if !$format;
1586         $opts .= ",format=alloc-track,file.driver=$format";
1587     } elsif ($format) {
1588         $opts .= ",format=$format";
1589     }
1590
1591     my $cache_direct = 0;
1592
1593     if (my $cache = $drive->{cache}) {
1594         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1595     } elsif (!drive_is_cdrom($drive)) {
1596         $opts .= ",cache=none";
1597         $cache_direct = 1;
1598     }
1599
1600     # aio native works only with O_DIRECT
1601     if (!$drive->{aio}) {
1602         if($cache_direct) {
1603             $opts .= ",aio=native";
1604         } else {
1605             $opts .= ",aio=threads";
1606         }
1607     }
1608
1609     if (!drive_is_cdrom($drive)) {
1610         my $detectzeroes;
1611         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1612             $detectzeroes = 'off';
1613         } elsif ($drive->{discard}) {
1614             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1615         } else {
1616             # This used to be our default with discard not being specified:
1617             $detectzeroes = 'on';
1618         }
1619
1620         # note: 'detect-zeroes' works per blockdev and we want it to persist
1621         # after the alloc-track is removed, so put it on 'file' directly
1622         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1623         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1624     }
1625
1626     if ($pbs_name) {
1627         $opts .= ",backing=$pbs_name";
1628         $opts .= ",auto-remove=on";
1629     }
1630
1631     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1632     my $file_param = "file";
1633     if ($pbs_name) {
1634         # non-rbd drivers require the underlying file to be a seperate block
1635         # node, so add a second .file indirection
1636         $file_param .= ".file" if !$is_rbd;
1637         $file_param .= ".filename";
1638     }
1639     my $pathinfo = $path ? "$file_param=$path," : '';
1640
1641     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1642 }
1643
1644 sub print_pbs_blockdev {
1645     my ($pbs_conf, $pbs_name) = @_;
1646     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1647     $blockdev .= ",repository=$pbs_conf->{repository}";
1648     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1649     $blockdev .= ",archive=$pbs_conf->{archive}";
1650     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1651     return $blockdev;
1652 }
1653
1654 sub print_netdevice_full {
1655     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1656
1657     my $device = $net->{model};
1658     if ($net->{model} eq 'virtio') {
1659          $device = 'virtio-net-pci';
1660      };
1661
1662     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1663     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1664     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1665         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1666         # and out of each queue plus one config interrupt and control vector queue
1667         my $vectors = $net->{queues} * 2 + 2;
1668         $tmpstr .= ",vectors=$vectors,mq=on";
1669     }
1670     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1671
1672     if (my $mtu = $net->{mtu}) {
1673         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1674             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1675             if ($mtu == 1) {
1676                  $mtu = $bridge_mtu;
1677             } elsif ($mtu < 576) {
1678                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1679             } elsif ($mtu > $bridge_mtu) {
1680                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1681             }
1682             $tmpstr .= ",host_mtu=$mtu";
1683         } else {
1684             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1685         }
1686     }
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     if (is_native($arch)) {
1725         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1726     }
1727
1728     my $vmname = $conf->{name} || "vm$vmid";
1729
1730     my $netdev = "";
1731     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733     if ($net->{bridge}) {
1734         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1735             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1736     } else {
1737         $netdev = "type=user,id=$netid,hostname=$vmname";
1738     }
1739
1740     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1741
1742     return $netdev;
1743 }
1744
1745 my $vga_map = {
1746     'cirrus' => 'cirrus-vga',
1747     'std' => 'VGA',
1748     'vmware' => 'vmware-svga',
1749     'virtio' => 'virtio-vga',
1750 };
1751
1752 sub print_vga_device {
1753     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1754
1755     my $type = $vga_map->{$vga->{type}};
1756     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1757         $type = 'virtio-gpu';
1758     }
1759     my $vgamem_mb = $vga->{memory};
1760
1761     my $max_outputs = '';
1762     if ($qxlnum) {
1763         $type = $id ? 'qxl' : 'qxl-vga';
1764
1765         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1766             # set max outputs so linux can have up to 4 qxl displays with one device
1767             if (min_version($machine_version, 4, 1)) {
1768                 $max_outputs = ",max_outputs=4";
1769             }
1770         }
1771     }
1772
1773     die "no devicetype for $vga->{type}\n" if !$type;
1774
1775     my $memory = "";
1776     if ($vgamem_mb) {
1777         if ($vga->{type} eq 'virtio') {
1778             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1779             $memory = ",max_hostmem=$bytes";
1780         } elsif ($qxlnum) {
1781             # from https://www.spice-space.org/multiple-monitors.html
1782             $memory = ",vgamem_mb=$vga->{memory}";
1783             my $ram = $vgamem_mb * 4;
1784             my $vram = $vgamem_mb * 2;
1785             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1786         } else {
1787             $memory = ",vgamem_mb=$vga->{memory}";
1788         }
1789     } elsif ($qxlnum && $id) {
1790         $memory = ",ram_size=67108864,vram_size=33554432";
1791     }
1792
1793     my $edidoff = "";
1794     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1795         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1796     }
1797
1798     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1799     my $vgaid = "vga" . ($id // '');
1800     my $pciaddr;
1801
1802     if ($q35 && $vgaid eq 'vga') {
1803         # the first display uses pcie.0 bus on q35 machines
1804         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1805     } else {
1806         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1807     }
1808
1809     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1810 }
1811
1812 sub parse_number_sets {
1813     my ($set) = @_;
1814     my $res = [];
1815     foreach my $part (split(/;/, $set)) {
1816         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818             push @$res, [ $1, $2 ];
1819         } else {
1820             die "invalid range: $part\n";
1821         }
1822     }
1823     return $res;
1824 }
1825
1826 sub parse_numa {
1827     my ($data) = @_;
1828
1829     my $res = parse_property_string($numa_fmt, $data);
1830     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1831     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1832     return $res;
1833 }
1834
1835 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1836 sub parse_net {
1837     my ($data) = @_;
1838
1839     my $res = eval { parse_property_string($net_fmt, $data) };
1840     if ($@) {
1841         warn $@;
1842         return;
1843     }
1844     if (!defined($res->{macaddr})) {
1845         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1846         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1847     }
1848     return $res;
1849 }
1850
1851 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1852 sub parse_ipconfig {
1853     my ($data) = @_;
1854
1855     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1856     if ($@) {
1857         warn $@;
1858         return;
1859     }
1860
1861     if ($res->{gw} && !$res->{ip}) {
1862         warn 'gateway specified without specifying an IP address';
1863         return;
1864     }
1865     if ($res->{gw6} && !$res->{ip6}) {
1866         warn 'IPv6 gateway specified without specifying an IPv6 address';
1867         return;
1868     }
1869     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1870         warn 'gateway specified together with DHCP';
1871         return;
1872     }
1873     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1874         # gw6 + auto/dhcp
1875         warn "IPv6 gateway specified together with $res->{ip6} address";
1876         return;
1877     }
1878
1879     if (!$res->{ip} && !$res->{ip6}) {
1880         return { ip => 'dhcp', ip6 => 'dhcp' };
1881     }
1882
1883     return $res;
1884 }
1885
1886 sub print_net {
1887     my $net = shift;
1888
1889     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1890 }
1891
1892 sub add_random_macs {
1893     my ($settings) = @_;
1894
1895     foreach my $opt (keys %$settings) {
1896         next if $opt !~ m/^net(\d+)$/;
1897         my $net = parse_net($settings->{$opt});
1898         next if !$net;
1899         $settings->{$opt} = print_net($net);
1900     }
1901 }
1902
1903 sub vm_is_volid_owner {
1904     my ($storecfg, $vmid, $volid) = @_;
1905
1906     if ($volid !~  m|^/|) {
1907         my ($path, $owner);
1908         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1909         if ($owner && ($owner == $vmid)) {
1910             return 1;
1911         }
1912     }
1913
1914     return;
1915 }
1916
1917 sub vmconfig_register_unused_drive {
1918     my ($storecfg, $vmid, $conf, $drive) = @_;
1919
1920     if (drive_is_cloudinit($drive)) {
1921         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1922         warn $@ if $@;
1923     } elsif (!drive_is_cdrom($drive)) {
1924         my $volid = $drive->{file};
1925         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1926             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1927         }
1928     }
1929 }
1930
1931 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1932 my $smbios1_fmt = {
1933     uuid => {
1934         type => 'string',
1935         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1936         format_description => 'UUID',
1937         description => "Set SMBIOS1 UUID.",
1938         optional => 1,
1939     },
1940     version => {
1941         type => 'string',
1942         pattern => '[A-Za-z0-9+\/]+={0,2}',
1943         format_description => 'Base64 encoded string',
1944         description => "Set SMBIOS1 version.",
1945         optional => 1,
1946     },
1947     serial => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 serial number.",
1952         optional => 1,
1953     },
1954     manufacturer => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 manufacturer.",
1959         optional => 1,
1960     },
1961     product => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 product ID.",
1966         optional => 1,
1967     },
1968     sku => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 SKU string.",
1973         optional => 1,
1974     },
1975     family => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 family string.",
1980         optional => 1,
1981     },
1982     base64 => {
1983         type => 'boolean',
1984         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1985         optional => 1,
1986     },
1987 };
1988
1989 sub parse_smbios1 {
1990     my ($data) = @_;
1991
1992     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1993     warn $@ if $@;
1994     return $res;
1995 }
1996
1997 sub print_smbios1 {
1998     my ($smbios1) = @_;
1999     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2000 }
2001
2002 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2003
2004 sub parse_watchdog {
2005     my ($value) = @_;
2006
2007     return if !$value;
2008
2009     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2010     warn $@ if $@;
2011     return $res;
2012 }
2013
2014 sub parse_guest_agent {
2015     my ($conf) = @_;
2016
2017     return {} if !defined($conf->{agent});
2018
2019     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2020     warn $@ if $@;
2021
2022     # if the agent is disabled ignore the other potentially set properties
2023     return {} if !$res->{enabled};
2024     return $res;
2025 }
2026
2027 sub get_qga_key {
2028     my ($conf, $key) = @_;
2029     return undef if !defined($conf->{agent});
2030
2031     my $agent = parse_guest_agent($conf);
2032     return $agent->{$key};
2033 }
2034
2035 sub parse_vga {
2036     my ($value) = @_;
2037
2038     return {} if !$value;
2039     my $res = eval { parse_property_string($vga_fmt, $value) };
2040     warn $@ if $@;
2041     return $res;
2042 }
2043
2044 sub parse_rng {
2045     my ($value) = @_;
2046
2047     return if !$value;
2048
2049     my $res = eval { parse_property_string($rng_fmt, $value) };
2050     warn $@ if $@;
2051     return $res;
2052 }
2053
2054 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2055 sub verify_usb_device {
2056     my ($value, $noerr) = @_;
2057
2058     return $value if parse_usb_device($value);
2059
2060     return if $noerr;
2061
2062     die "unable to parse usb device\n";
2063 }
2064
2065 # add JSON properties for create and set function
2066 sub json_config_properties {
2067     my $prop = shift;
2068
2069     foreach my $opt (keys %$confdesc) {
2070         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2071             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2072         $prop->{$opt} = $confdesc->{$opt};
2073     }
2074
2075     return $prop;
2076 }
2077
2078 # return copy of $confdesc_cloudinit to generate documentation
2079 sub cloudinit_config_properties {
2080
2081     return dclone($confdesc_cloudinit);
2082 }
2083
2084 sub check_type {
2085     my ($key, $value) = @_;
2086
2087     die "unknown setting '$key'\n" if !$confdesc->{$key};
2088
2089     my $type = $confdesc->{$key}->{type};
2090
2091     if (!defined($value)) {
2092         die "got undefined value\n";
2093     }
2094
2095     if ($value =~ m/[\n\r]/) {
2096         die "property contains a line feed\n";
2097     }
2098
2099     if ($type eq 'boolean') {
2100         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2101         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2102         die "type check ('boolean') failed - got '$value'\n";
2103     } elsif ($type eq 'integer') {
2104         return int($1) if $value =~ m/^(\d+)$/;
2105         die "type check ('integer') failed - got '$value'\n";
2106     } elsif ($type eq 'number') {
2107         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2108         die "type check ('number') failed - got '$value'\n";
2109     } elsif ($type eq 'string') {
2110         if (my $fmt = $confdesc->{$key}->{format}) {
2111             PVE::JSONSchema::check_format($fmt, $value);
2112             return $value;
2113         }
2114         $value =~ s/^\"(.*)\"$/$1/;
2115         return $value;
2116     } else {
2117         die "internal error"
2118     }
2119 }
2120
2121 sub destroy_vm {
2122     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2123
2124     my $conf = PVE::QemuConfig->load_config($vmid);
2125
2126     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2127
2128     if ($conf->{template}) {
2129         # check if any base image is still used by a linked clone
2130         PVE::QemuConfig->foreach_volume($conf, sub {
2131                 my ($ds, $drive) = @_;
2132                 return if drive_is_cdrom($drive);
2133
2134                 my $volid = $drive->{file};
2135                 return if !$volid || $volid =~ m|^/|;
2136
2137                 die "base volume '$volid' is still in use by linked cloned\n"
2138                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2139
2140         });
2141     }
2142
2143     # only remove disks owned by this VM (referenced in the config)
2144     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2145         my ($ds, $drive) = @_;
2146         return if drive_is_cdrom($drive, 1);
2147
2148         my $volid = $drive->{file};
2149         return if !$volid || $volid =~ m|^/|;
2150
2151         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2152         return if !$path || !$owner || ($owner != $vmid);
2153
2154         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2155         warn "Could not remove disk '$volid', check manually: $@" if $@;
2156     });
2157
2158     if ($purge_unreferenced) { # also remove unreferenced disk
2159         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2160         PVE::Storage::foreach_volid($vmdisks, sub {
2161             my ($volid, $sid, $volname, $d) = @_;
2162             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2163             warn $@ if $@;
2164         });
2165     }
2166
2167     if (defined $replacement_conf) {
2168         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2169     } else {
2170         PVE::QemuConfig->destroy_config($vmid);
2171     }
2172 }
2173
2174 sub parse_vm_config {
2175     my ($filename, $raw) = @_;
2176
2177     return if !defined($raw);
2178
2179     my $res = {
2180         digest => Digest::SHA::sha1_hex($raw),
2181         snapshots => {},
2182         pending => {},
2183     };
2184
2185     $filename =~ m|/qemu-server/(\d+)\.conf$|
2186         || die "got strange filename '$filename'";
2187
2188     my $vmid = $1;
2189
2190     my $conf = $res;
2191     my $descr;
2192     my $section = '';
2193
2194     my @lines = split(/\n/, $raw);
2195     foreach my $line (@lines) {
2196         next if $line =~ m/^\s*$/;
2197
2198         if ($line =~ m/^\[PENDING\]\s*$/i) {
2199             $section = 'pending';
2200             if (defined($descr)) {
2201                 $descr =~ s/\s+$//;
2202                 $conf->{description} = $descr;
2203             }
2204             $descr = undef;
2205             $conf = $res->{$section} = {};
2206             next;
2207
2208         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2209             $section = $1;
2210             if (defined($descr)) {
2211                 $descr =~ s/\s+$//;
2212                 $conf->{description} = $descr;
2213             }
2214             $descr = undef;
2215             $conf = $res->{snapshots}->{$section} = {};
2216             next;
2217         }
2218
2219         if ($line =~ m/^\#(.*)\s*$/) {
2220             $descr = '' if !defined($descr);
2221             $descr .= PVE::Tools::decode_text($1) . "\n";
2222             next;
2223         }
2224
2225         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2226             $descr = '' if !defined($descr);
2227             $descr .= PVE::Tools::decode_text($2);
2228         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2229             $conf->{snapstate} = $1;
2230         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2231             my $key = $1;
2232             my $value = $2;
2233             $conf->{$key} = $value;
2234         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2235             my $value = $1;
2236             if ($section eq 'pending') {
2237                 $conf->{delete} = $value; # we parse this later
2238             } else {
2239                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2240             }
2241         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2242             my $key = $1;
2243             my $value = $2;
2244             eval { $value = check_type($key, $value); };
2245             if ($@) {
2246                 warn "vm $vmid - unable to parse value of '$key' - $@";
2247             } else {
2248                 $key = 'ide2' if $key eq 'cdrom';
2249                 my $fmt = $confdesc->{$key}->{format};
2250                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2251                     my $v = parse_drive($key, $value);
2252                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2253                         $v->{file} = $volid;
2254                         $value = print_drive($v);
2255                     } else {
2256                         warn "vm $vmid - unable to parse value of '$key'\n";
2257                         next;
2258                     }
2259                 }
2260
2261                 $conf->{$key} = $value;
2262             }
2263         } else {
2264             warn "vm $vmid - unable to parse config: $line\n";
2265         }
2266     }
2267
2268     if (defined($descr)) {
2269         $descr =~ s/\s+$//;
2270         $conf->{description} = $descr;
2271     }
2272     delete $res->{snapstate}; # just to be sure
2273
2274     return $res;
2275 }
2276
2277 sub write_vm_config {
2278     my ($filename, $conf) = @_;
2279
2280     delete $conf->{snapstate}; # just to be sure
2281
2282     if ($conf->{cdrom}) {
2283         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2284         $conf->{ide2} = $conf->{cdrom};
2285         delete $conf->{cdrom};
2286     }
2287
2288     # we do not use 'smp' any longer
2289     if ($conf->{sockets}) {
2290         delete $conf->{smp};
2291     } elsif ($conf->{smp}) {
2292         $conf->{sockets} = $conf->{smp};
2293         delete $conf->{cores};
2294         delete $conf->{smp};
2295     }
2296
2297     my $used_volids = {};
2298
2299     my $cleanup_config = sub {
2300         my ($cref, $pending, $snapname) = @_;
2301
2302         foreach my $key (keys %$cref) {
2303             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2304                 $key eq 'snapstate' || $key eq 'pending';
2305             my $value = $cref->{$key};
2306             if ($key eq 'delete') {
2307                 die "propertry 'delete' is only allowed in [PENDING]\n"
2308                     if !$pending;
2309                 # fixme: check syntax?
2310                 next;
2311             }
2312             eval { $value = check_type($key, $value); };
2313             die "unable to parse value of '$key' - $@" if $@;
2314
2315             $cref->{$key} = $value;
2316
2317             if (!$snapname && is_valid_drivename($key)) {
2318                 my $drive = parse_drive($key, $value);
2319                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2320             }
2321         }
2322     };
2323
2324     &$cleanup_config($conf);
2325
2326     &$cleanup_config($conf->{pending}, 1);
2327
2328     foreach my $snapname (keys %{$conf->{snapshots}}) {
2329         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2330         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2331     }
2332
2333     # remove 'unusedX' settings if we re-add a volume
2334     foreach my $key (keys %$conf) {
2335         my $value = $conf->{$key};
2336         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2337             delete $conf->{$key};
2338         }
2339     }
2340
2341     my $generate_raw_config = sub {
2342         my ($conf, $pending) = @_;
2343
2344         my $raw = '';
2345
2346         # add description as comment to top of file
2347         if (defined(my $descr = $conf->{description})) {
2348             if ($descr) {
2349                 foreach my $cl (split(/\n/, $descr)) {
2350                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2351                 }
2352             } else {
2353                 $raw .= "#\n" if $pending;
2354             }
2355         }
2356
2357         foreach my $key (sort keys %$conf) {
2358             next if $key =~ /^(digest|description|pending|snapshots)$/;
2359             $raw .= "$key: $conf->{$key}\n";
2360         }
2361         return $raw;
2362     };
2363
2364     my $raw = &$generate_raw_config($conf);
2365
2366     if (scalar(keys %{$conf->{pending}})){
2367         $raw .= "\n[PENDING]\n";
2368         $raw .= &$generate_raw_config($conf->{pending}, 1);
2369     }
2370
2371     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2372         $raw .= "\n[$snapname]\n";
2373         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2374     }
2375
2376     return $raw;
2377 }
2378
2379 sub load_defaults {
2380
2381     my $res = {};
2382
2383     # we use static defaults from our JSON schema configuration
2384     foreach my $key (keys %$confdesc) {
2385         if (defined(my $default = $confdesc->{$key}->{default})) {
2386             $res->{$key} = $default;
2387         }
2388     }
2389
2390     return $res;
2391 }
2392
2393 sub config_list {
2394     my $vmlist = PVE::Cluster::get_vmlist();
2395     my $res = {};
2396     return $res if !$vmlist || !$vmlist->{ids};
2397     my $ids = $vmlist->{ids};
2398     my $nodename = nodename();
2399
2400     foreach my $vmid (keys %$ids) {
2401         my $d = $ids->{$vmid};
2402         next if !$d->{node} || $d->{node} ne $nodename;
2403         next if !$d->{type} || $d->{type} ne 'qemu';
2404         $res->{$vmid}->{exists} = 1;
2405     }
2406     return $res;
2407 }
2408
2409 # test if VM uses local resources (to prevent migration)
2410 sub check_local_resources {
2411     my ($conf, $noerr) = @_;
2412
2413     my @loc_res = ();
2414
2415     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2416     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2417
2418     push @loc_res, "ivshmem" if $conf->{ivshmem};
2419
2420     foreach my $k (keys %$conf) {
2421         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2422         # sockets are safe: they will recreated be on the target side post-migrate
2423         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2424         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2425     }
2426
2427     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2428
2429     return \@loc_res;
2430 }
2431
2432 # check if used storages are available on all nodes (use by migrate)
2433 sub check_storage_availability {
2434     my ($storecfg, $conf, $node) = @_;
2435
2436     PVE::QemuConfig->foreach_volume($conf, sub {
2437         my ($ds, $drive) = @_;
2438
2439         my $volid = $drive->{file};
2440         return if !$volid;
2441
2442         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2443         return if !$sid;
2444
2445         # check if storage is available on both nodes
2446         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2447         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2448    });
2449 }
2450
2451 # list nodes where all VM images are available (used by has_feature API)
2452 sub shared_nodes {
2453     my ($conf, $storecfg) = @_;
2454
2455     my $nodelist = PVE::Cluster::get_nodelist();
2456     my $nodehash = { map { $_ => 1 } @$nodelist };
2457     my $nodename = nodename();
2458
2459     PVE::QemuConfig->foreach_volume($conf, sub {
2460         my ($ds, $drive) = @_;
2461
2462         my $volid = $drive->{file};
2463         return if !$volid;
2464
2465         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2466         if ($storeid) {
2467             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2468             if ($scfg->{disable}) {
2469                 $nodehash = {};
2470             } elsif (my $avail = $scfg->{nodes}) {
2471                 foreach my $node (keys %$nodehash) {
2472                     delete $nodehash->{$node} if !$avail->{$node};
2473                 }
2474             } elsif (!$scfg->{shared}) {
2475                 foreach my $node (keys %$nodehash) {
2476                     delete $nodehash->{$node} if $node ne $nodename
2477                 }
2478             }
2479         }
2480     });
2481
2482     return $nodehash
2483 }
2484
2485 sub check_local_storage_availability {
2486     my ($conf, $storecfg) = @_;
2487
2488     my $nodelist = PVE::Cluster::get_nodelist();
2489     my $nodehash = { map { $_ => {} } @$nodelist };
2490
2491     PVE::QemuConfig->foreach_volume($conf, sub {
2492         my ($ds, $drive) = @_;
2493
2494         my $volid = $drive->{file};
2495         return if !$volid;
2496
2497         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2498         if ($storeid) {
2499             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2500
2501             if ($scfg->{disable}) {
2502                 foreach my $node (keys %$nodehash) {
2503                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2504                 }
2505             } elsif (my $avail = $scfg->{nodes}) {
2506                 foreach my $node (keys %$nodehash) {
2507                     if (!$avail->{$node}) {
2508                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2509                     }
2510                 }
2511             }
2512         }
2513     });
2514
2515     foreach my $node (values %$nodehash) {
2516         if (my $unavail = $node->{unavailable_storages}) {
2517             $node->{unavailable_storages} = [ sort keys %$unavail ];
2518         }
2519     }
2520
2521     return $nodehash
2522 }
2523
2524 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2525 sub check_running {
2526     my ($vmid, $nocheck, $node) = @_;
2527
2528     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2529     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2530 }
2531
2532 sub vzlist {
2533
2534     my $vzlist = config_list();
2535
2536     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2537
2538     while (defined(my $de = $fd->read)) {
2539         next if $de !~ m/^(\d+)\.pid$/;
2540         my $vmid = $1;
2541         next if !defined($vzlist->{$vmid});
2542         if (my $pid = check_running($vmid)) {
2543             $vzlist->{$vmid}->{pid} = $pid;
2544         }
2545     }
2546
2547     return $vzlist;
2548 }
2549
2550 our $vmstatus_return_properties = {
2551     vmid => get_standard_option('pve-vmid'),
2552     status => {
2553         description => "Qemu process status.",
2554         type => 'string',
2555         enum => ['stopped', 'running'],
2556     },
2557     maxmem => {
2558         description => "Maximum memory in bytes.",
2559         type => 'integer',
2560         optional => 1,
2561         renderer => 'bytes',
2562     },
2563     maxdisk => {
2564         description => "Root disk size in bytes.",
2565         type => 'integer',
2566         optional => 1,
2567         renderer => 'bytes',
2568     },
2569     name => {
2570         description => "VM name.",
2571         type => 'string',
2572         optional => 1,
2573     },
2574     qmpstatus => {
2575         description => "Qemu QMP agent status.",
2576         type => 'string',
2577         optional => 1,
2578     },
2579     pid => {
2580         description => "PID of running qemu process.",
2581         type => 'integer',
2582         optional => 1,
2583     },
2584     uptime => {
2585         description => "Uptime.",
2586         type => 'integer',
2587         optional => 1,
2588         renderer => 'duration',
2589     },
2590     cpus => {
2591         description => "Maximum usable CPUs.",
2592         type => 'number',
2593         optional => 1,
2594     },
2595     lock => {
2596         description => "The current config lock, if any.",
2597         type => 'string',
2598         optional => 1,
2599     },
2600     tags => {
2601         description  => "The current configured tags, if any",
2602         type => 'string',
2603         optional => 1,
2604     },
2605     'running-machine' => {
2606         description => "The currently running machine type (if running).",
2607         type => 'string',
2608         optional => 1,
2609     },
2610     'running-qemu' => {
2611         description => "The currently running QEMU version (if running).",
2612         type => 'string',
2613         optional => 1,
2614     },
2615 };
2616
2617 my $last_proc_pid_stat;
2618
2619 # get VM status information
2620 # This must be fast and should not block ($full == false)
2621 # We only query KVM using QMP if $full == true (this can be slow)
2622 sub vmstatus {
2623     my ($opt_vmid, $full) = @_;
2624
2625     my $res = {};
2626
2627     my $storecfg = PVE::Storage::config();
2628
2629     my $list = vzlist();
2630     my $defaults = load_defaults();
2631
2632     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2633
2634     my $cpucount = $cpuinfo->{cpus} || 1;
2635
2636     foreach my $vmid (keys %$list) {
2637         next if $opt_vmid && ($vmid ne $opt_vmid);
2638
2639         my $conf = PVE::QemuConfig->load_config($vmid);
2640
2641         my $d = { vmid => $vmid };
2642         $d->{pid} = $list->{$vmid}->{pid};
2643
2644         # fixme: better status?
2645         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2646
2647         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2648         if (defined($size)) {
2649             $d->{disk} = 0; # no info available
2650             $d->{maxdisk} = $size;
2651         } else {
2652             $d->{disk} = 0;
2653             $d->{maxdisk} = 0;
2654         }
2655
2656         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2657             * ($conf->{cores} || $defaults->{cores});
2658         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2659         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2660
2661         $d->{name} = $conf->{name} || "VM $vmid";
2662         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2663             : $defaults->{memory}*(1024*1024);
2664
2665         if ($conf->{balloon}) {
2666             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2667             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2668                 : $defaults->{shares};
2669         }
2670
2671         $d->{uptime} = 0;
2672         $d->{cpu} = 0;
2673         $d->{mem} = 0;
2674
2675         $d->{netout} = 0;
2676         $d->{netin} = 0;
2677
2678         $d->{diskread} = 0;
2679         $d->{diskwrite} = 0;
2680
2681         $d->{template} = PVE::QemuConfig->is_template($conf);
2682
2683         $d->{serial} = 1 if conf_has_serial($conf);
2684         $d->{lock} = $conf->{lock} if $conf->{lock};
2685         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2686
2687         $res->{$vmid} = $d;
2688     }
2689
2690     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2691     foreach my $dev (keys %$netdev) {
2692         next if $dev !~ m/^tap([1-9]\d*)i/;
2693         my $vmid = $1;
2694         my $d = $res->{$vmid};
2695         next if !$d;
2696
2697         $d->{netout} += $netdev->{$dev}->{receive};
2698         $d->{netin} += $netdev->{$dev}->{transmit};
2699
2700         if ($full) {
2701             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2702             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2703         }
2704
2705     }
2706
2707     my $ctime = gettimeofday;
2708
2709     foreach my $vmid (keys %$list) {
2710
2711         my $d = $res->{$vmid};
2712         my $pid = $d->{pid};
2713         next if !$pid;
2714
2715         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2716         next if !$pstat; # not running
2717
2718         my $used = $pstat->{utime} + $pstat->{stime};
2719
2720         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2721
2722         if ($pstat->{vsize}) {
2723             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2724         }
2725
2726         my $old = $last_proc_pid_stat->{$pid};
2727         if (!$old) {
2728             $last_proc_pid_stat->{$pid} = {
2729                 time => $ctime,
2730                 used => $used,
2731                 cpu => 0,
2732             };
2733             next;
2734         }
2735
2736         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2737
2738         if ($dtime > 1000) {
2739             my $dutime = $used -  $old->{used};
2740
2741             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2742             $last_proc_pid_stat->{$pid} = {
2743                 time => $ctime,
2744                 used => $used,
2745                 cpu => $d->{cpu},
2746             };
2747         } else {
2748             $d->{cpu} = $old->{cpu};
2749         }
2750     }
2751
2752     return $res if !$full;
2753
2754     my $qmpclient = PVE::QMPClient->new();
2755
2756     my $ballooncb = sub {
2757         my ($vmid, $resp) = @_;
2758
2759         my $info = $resp->{'return'};
2760         return if !$info->{max_mem};
2761
2762         my $d = $res->{$vmid};
2763
2764         # use memory assigned to VM
2765         $d->{maxmem} = $info->{max_mem};
2766         $d->{balloon} = $info->{actual};
2767
2768         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2769             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2770             $d->{freemem} = $info->{free_mem};
2771         }
2772
2773         $d->{ballooninfo} = $info;
2774     };
2775
2776     my $blockstatscb = sub {
2777         my ($vmid, $resp) = @_;
2778         my $data = $resp->{'return'} || [];
2779         my $totalrdbytes = 0;
2780         my $totalwrbytes = 0;
2781
2782         for my $blockstat (@$data) {
2783             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2784             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2785
2786             $blockstat->{device} =~ s/drive-//;
2787             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2788         }
2789         $res->{$vmid}->{diskread} = $totalrdbytes;
2790         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2791     };
2792
2793     my $machinecb = sub {
2794         my ($vmid, $resp) = @_;
2795         my $data = $resp->{'return'} || [];
2796
2797         $res->{$vmid}->{'running-machine'} =
2798             PVE::QemuServer::Machine::current_from_query_machines($data);
2799     };
2800
2801     my $versioncb = sub {
2802         my ($vmid, $resp) = @_;
2803         my $data = $resp->{'return'} // {};
2804         my $version = 'unknown';
2805
2806         if (my $v = $data->{qemu}) {
2807             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2808         }
2809
2810         $res->{$vmid}->{'running-qemu'} = $version;
2811     };
2812
2813     my $statuscb = sub {
2814         my ($vmid, $resp) = @_;
2815
2816         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2817         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2818         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2819         # this fails if ballon driver is not loaded, so this must be
2820         # the last commnand (following command are aborted if this fails).
2821         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2822
2823         my $status = 'unknown';
2824         if (!defined($status = $resp->{'return'}->{status})) {
2825             warn "unable to get VM status\n";
2826             return;
2827         }
2828
2829         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2830     };
2831
2832     foreach my $vmid (keys %$list) {
2833         next if $opt_vmid && ($vmid ne $opt_vmid);
2834         next if !$res->{$vmid}->{pid}; # not running
2835         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2836     }
2837
2838     $qmpclient->queue_execute(undef, 2);
2839
2840     foreach my $vmid (keys %$list) {
2841         next if $opt_vmid && ($vmid ne $opt_vmid);
2842         next if !$res->{$vmid}->{pid}; #not running
2843
2844         # we can't use the $qmpclient since it might have already aborted on
2845         # 'query-balloon', but this might also fail for older versions...
2846         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2847         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2848     }
2849
2850     foreach my $vmid (keys %$list) {
2851         next if $opt_vmid && ($vmid ne $opt_vmid);
2852         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2853     }
2854
2855     return $res;
2856 }
2857
2858 sub conf_has_serial {
2859     my ($conf) = @_;
2860
2861     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2862         if ($conf->{"serial$i"}) {
2863             return 1;
2864         }
2865     }
2866
2867     return 0;
2868 }
2869
2870 sub conf_has_audio {
2871     my ($conf, $id) = @_;
2872
2873     $id //= 0;
2874     my $audio = $conf->{"audio$id"};
2875     return if !defined($audio);
2876
2877     my $audioproperties = parse_property_string($audio_fmt, $audio);
2878     my $audiodriver = $audioproperties->{driver} // 'spice';
2879
2880     return {
2881         dev => $audioproperties->{device},
2882         dev_id => "audiodev$id",
2883         backend => $audiodriver,
2884         backend_id => "$audiodriver-backend${id}",
2885     };
2886 }
2887
2888 sub audio_devs {
2889     my ($audio, $audiopciaddr, $machine_version) = @_;
2890
2891     my $devs = [];
2892
2893     my $id = $audio->{dev_id};
2894     my $audiodev = "";
2895     if (min_version($machine_version, 4, 2)) {
2896         $audiodev = ",audiodev=$audio->{backend_id}";
2897     }
2898
2899     if ($audio->{dev} eq 'AC97') {
2900         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2901     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2902         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2903         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2904         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2905     } else {
2906         die "unkown audio device '$audio->{dev}', implement me!";
2907     }
2908
2909     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2910
2911     return $devs;
2912 }
2913
2914 sub vga_conf_has_spice {
2915     my ($vga) = @_;
2916
2917     my $vgaconf = parse_vga($vga);
2918     my $vgatype = $vgaconf->{type};
2919     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2920
2921     return $1 || 1;
2922 }
2923
2924 sub is_native($) {
2925     my ($arch) = @_;
2926     return get_host_arch() eq $arch;
2927 }
2928
2929 sub get_vm_arch {
2930     my ($conf) = @_;
2931     return $conf->{arch} // get_host_arch();
2932 }
2933
2934 my $default_machines = {
2935     x86_64 => 'pc',
2936     aarch64 => 'virt',
2937 };
2938
2939 sub get_installed_machine_version {
2940     my ($kvmversion) = @_;
2941     $kvmversion = kvm_user_version() if !defined($kvmversion);
2942     $kvmversion =~ m/^(\d+\.\d+)/;
2943     return $1;
2944 }
2945
2946 sub windows_get_pinned_machine_version {
2947     my ($machine, $base_version, $kvmversion) = @_;
2948
2949     my $pin_version = $base_version;
2950     if (!defined($base_version) ||
2951         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2952     ) {
2953         $pin_version = get_installed_machine_version($kvmversion);
2954     }
2955     if (!$machine || $machine eq 'pc') {
2956         $machine = "pc-i440fx-$pin_version";
2957     } elsif ($machine eq 'q35') {
2958         $machine = "pc-q35-$pin_version";
2959     } elsif ($machine eq 'virt') {
2960         $machine = "virt-$pin_version";
2961     } else {
2962         warn "unknown machine type '$machine', not touching that!\n";
2963     }
2964
2965     return $machine;
2966 }
2967
2968 sub get_vm_machine {
2969     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2970
2971     my $machine = $forcemachine || $conf->{machine};
2972
2973     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2974         $kvmversion //= kvm_user_version();
2975         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2976         # layout which confuses windows quite a bit and may result in various regressions..
2977         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2978         if (windows_version($conf->{ostype})) {
2979             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2980         }
2981         $arch //= 'x86_64';
2982         $machine ||= $default_machines->{$arch};
2983         if ($add_pve_version) {
2984             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2985             $machine .= "+pve$pvever";
2986         }
2987     }
2988
2989     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
2990         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
2991         $machine = $1 if $is_pxe;
2992
2993         # for version-pinned machines that do not include a pve-version (e.g.
2994         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2995         $machine .= '+pve0';
2996
2997         $machine .= '.pxe' if $is_pxe;
2998     }
2999
3000     return $machine;
3001 }
3002
3003 sub get_ovmf_files($) {
3004     my ($arch) = @_;
3005
3006     my $ovmf = $OVMF->{$arch}
3007         or die "no OVMF images known for architecture '$arch'\n";
3008
3009     return @$ovmf;
3010 }
3011
3012 my $Arch2Qemu = {
3013     aarch64 => '/usr/bin/qemu-system-aarch64',
3014     x86_64 => '/usr/bin/qemu-system-x86_64',
3015 };
3016 sub get_command_for_arch($) {
3017     my ($arch) = @_;
3018     return '/usr/bin/kvm' if is_native($arch);
3019
3020     my $cmd = $Arch2Qemu->{$arch}
3021         or die "don't know how to emulate architecture '$arch'\n";
3022     return $cmd;
3023 }
3024
3025 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3026 # to use in a QEMU command line (-cpu element), first array_intersect the result
3027 # of query_supported_ with query_understood_. This is necessary because:
3028 #
3029 # a) query_understood_ returns flags the host cannot use and
3030 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3031 #    flags, but CPU settings - with most of them being flags. Those settings
3032 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3033 #
3034 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3035 # expensive. If you need the value returned from this, you can get it much
3036 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3037 # $accel being 'kvm' or 'tcg'.
3038 #
3039 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3040 # changes, automatically populating pmxcfs.
3041 #
3042 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3043 # since kvm and tcg machines support different flags
3044 #
3045 sub query_supported_cpu_flags {
3046     my ($arch) = @_;
3047
3048     $arch //= get_host_arch();
3049     my $default_machine = $default_machines->{$arch};
3050
3051     my $flags = {};
3052
3053     # FIXME: Once this is merged, the code below should work for ARM as well:
3054     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3055     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3056         $arch eq "aarch64";
3057
3058     my $kvm_supported = defined(kvm_version());
3059     my $qemu_cmd = get_command_for_arch($arch);
3060     my $fakevmid = -1;
3061     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3062
3063     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3064     my $query_supported_run_qemu = sub {
3065         my ($kvm) = @_;
3066
3067         my $flags = {};
3068         my $cmd = [
3069             $qemu_cmd,
3070             '-machine', $default_machine,
3071             '-display', 'none',
3072             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3073             '-mon', 'chardev=qmp,mode=control',
3074             '-pidfile', $pidfile,
3075             '-S', '-daemonize'
3076         ];
3077
3078         if (!$kvm) {
3079             push @$cmd, '-accel', 'tcg';
3080         }
3081
3082         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3083         die "QEMU flag querying VM exited with code " . $rc if $rc;
3084
3085         eval {
3086             my $cmd_result = mon_cmd(
3087                 $fakevmid,
3088                 'query-cpu-model-expansion',
3089                 type => 'full',
3090                 model => { name => 'host' }
3091             );
3092
3093             my $props = $cmd_result->{model}->{props};
3094             foreach my $prop (keys %$props) {
3095                 next if $props->{$prop} ne '1';
3096                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3097                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3098                 # We only keep those with underscores, to match /proc/cpuinfo
3099                 $prop =~ s/\.|-/_/g;
3100                 $flags->{$prop} = 1;
3101             }
3102         };
3103         my $err = $@;
3104
3105         # force stop with 10 sec timeout and 'nocheck'
3106         # always stop, even if QMP failed
3107         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3108
3109         die $err if $err;
3110
3111         return [ sort keys %$flags ];
3112     };
3113
3114     # We need to query QEMU twice, since KVM and TCG have different supported flags
3115     PVE::QemuConfig->lock_config($fakevmid, sub {
3116         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3117         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3118
3119         if ($kvm_supported) {
3120             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3121             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3122         }
3123     });
3124
3125     return $flags;
3126 }
3127
3128 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3129 my $understood_cpu_flag_dir = "/usr/share/kvm";
3130 sub query_understood_cpu_flags {
3131     my $arch = get_host_arch();
3132     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3133
3134     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3135         if ! -e $filepath;
3136
3137     my $raw = file_get_contents($filepath);
3138     $raw =~ s/^\s+|\s+$//g;
3139     my @flags = split(/\s+/, $raw);
3140
3141     return \@flags;
3142 }
3143
3144 sub config_to_command {
3145     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3146         $pbs_backing) = @_;
3147
3148     my $cmd = [];
3149     my $globalFlags = [];
3150     my $machineFlags = [];
3151     my $rtcFlags = [];
3152     my $devices = [];
3153     my $pciaddr = '';
3154     my $bridges = {};
3155     my $ostype = $conf->{ostype};
3156     my $winversion = windows_version($ostype);
3157     my $kvm = $conf->{kvm};
3158     my $nodename = nodename();
3159
3160     my $arch = get_vm_arch($conf);
3161     my $kvm_binary = get_command_for_arch($arch);
3162     my $kvmver = kvm_user_version($kvm_binary);
3163
3164     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3165         $kvmver //= "undefined";
3166         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3167     }
3168
3169     my $add_pve_version = min_version($kvmver, 4, 1);
3170
3171     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3172     my $machine_version = extract_version($machine_type, $kvmver);
3173     $kvm //= 1 if is_native($arch);
3174
3175     $machine_version =~ m/(\d+)\.(\d+)/;
3176     my ($machine_major, $machine_minor) = ($1, $2);
3177
3178     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3179         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3180     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3181         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3182             ." please upgrade node '$nodename'\n"
3183     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3184         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3185         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3186             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3187             ." node '$nodename'\n";
3188     }
3189
3190     # if a specific +pve version is required for a feature, use $version_guard
3191     # instead of min_version to allow machines to be run with the minimum
3192     # required version
3193     my $required_pve_version = 0;
3194     my $version_guard = sub {
3195         my ($major, $minor, $pve) = @_;
3196         return 0 if !min_version($machine_version, $major, $minor, $pve);
3197         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3198         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3199         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3200         return 1;
3201     };
3202
3203     if ($kvm && !defined kvm_version()) {
3204         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3205             ." or enable in BIOS.\n";
3206     }
3207
3208     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3209     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3210     my $use_old_bios_files = undef;
3211     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3212
3213     my $cpuunits = defined($conf->{cpuunits}) ?
3214             $conf->{cpuunits} : $defaults->{cpuunits};
3215
3216     push @$cmd, $kvm_binary;
3217
3218     push @$cmd, '-id', $vmid;
3219
3220     my $vmname = $conf->{name} || "vm$vmid";
3221
3222     push @$cmd, '-name', $vmname;
3223
3224     push @$cmd, '-no-shutdown';
3225
3226     my $use_virtio = 0;
3227
3228     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3229     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3230     push @$cmd, '-mon', "chardev=qmp,mode=control";
3231
3232     if (min_version($machine_version, 2, 12)) {
3233         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3234         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3235     }
3236
3237     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3238
3239     push @$cmd, '-daemonize';
3240
3241     if ($conf->{smbios1}) {
3242         my $smbios_conf = parse_smbios1($conf->{smbios1});
3243         if ($smbios_conf->{base64}) {
3244             # Do not pass base64 flag to qemu
3245             delete $smbios_conf->{base64};
3246             my $smbios_string = "";
3247             foreach my $key (keys %$smbios_conf) {
3248                 my $value;
3249                 if ($key eq "uuid") {
3250                     $value = $smbios_conf->{uuid}
3251                 } else {
3252                     $value = decode_base64($smbios_conf->{$key});
3253                 }
3254                 # qemu accepts any binary data, only commas need escaping by double comma
3255                 $value =~ s/,/,,/g;
3256                 $smbios_string .= "," . $key . "=" . $value if $value;
3257             }
3258             push @$cmd, '-smbios', "type=1" . $smbios_string;
3259         } else {
3260             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3261         }
3262     }
3263
3264     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3265         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3266         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3267
3268         my ($path, $format);
3269         if (my $efidisk = $conf->{efidisk0}) {
3270             my $d = parse_drive('efidisk0', $efidisk);
3271             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3272             $format = $d->{format};
3273             if ($storeid) {
3274                 $path = PVE::Storage::path($storecfg, $d->{file});
3275                 if (!defined($format)) {
3276                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3277                     $format = qemu_img_format($scfg, $volname);
3278                 }
3279             } else {
3280                 $path = $d->{file};
3281                 die "efidisk format must be specified\n"
3282                     if !defined($format);
3283             }
3284         } else {
3285             warn "no efidisk configured! Using temporary efivars disk.\n";
3286             $path = "/tmp/$vmid-ovmf.fd";
3287             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3288             $format = 'raw';
3289         }
3290
3291         my $size_str = "";
3292
3293         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3294             $size_str = ",size=" . (-s $ovmf_vars);
3295         }
3296
3297         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3298         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3299     }
3300
3301     # load q35 config
3302     if ($q35) {
3303         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3304         if (min_version($machine_version, 4, 0)) {
3305             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3306         } else {
3307             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3308         }
3309     }
3310
3311     if ($conf->{vmgenid}) {
3312         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3313     }
3314
3315     # add usb controllers
3316     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3317         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3318     push @$devices, @usbcontrollers if @usbcontrollers;
3319     my $vga = parse_vga($conf->{vga});
3320
3321     my $qxlnum = vga_conf_has_spice($conf->{vga});
3322     $vga->{type} = 'qxl' if $qxlnum;
3323
3324     if (!$vga->{type}) {
3325         if ($arch eq 'aarch64') {
3326             $vga->{type} = 'virtio';
3327         } elsif (min_version($machine_version, 2, 9)) {
3328             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3329         } else {
3330             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3331         }
3332     }
3333
3334     # enable absolute mouse coordinates (needed by vnc)
3335     my $tablet;
3336     if (defined($conf->{tablet})) {
3337         $tablet = $conf->{tablet};
3338     } else {
3339         $tablet = $defaults->{tablet};
3340         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3341         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3342     }
3343
3344     if ($tablet) {
3345         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3346         my $kbd = print_keyboarddevice_full($conf, $arch);
3347         push @$devices, '-device', $kbd if defined($kbd);
3348     }
3349
3350     my $bootorder = device_bootorder($conf);
3351
3352     # host pci device passthrough
3353     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3354         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3355
3356     # usb devices
3357     my $usb_dev_features = {};
3358     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3359
3360     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3361         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3362     push @$devices, @usbdevices if @usbdevices;
3363
3364     # serial devices
3365     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3366         if (my $path = $conf->{"serial$i"}) {
3367             if ($path eq 'socket') {
3368                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3369                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3370                 # On aarch64, serial0 is the UART device. Qemu only allows
3371                 # connecting UART devices via the '-serial' command line, as
3372                 # the device has a fixed slot on the hardware...
3373                 if ($arch eq 'aarch64' && $i == 0) {
3374                     push @$devices, '-serial', "chardev:serial$i";
3375                 } else {
3376                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3377                 }
3378             } else {
3379                 die "no such serial device\n" if ! -c $path;
3380                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3381                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3382             }
3383         }
3384     }
3385
3386     # parallel devices
3387     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3388         if (my $path = $conf->{"parallel$i"}) {
3389             die "no such parallel device\n" if ! -c $path;
3390             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3391             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3392             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3393         }
3394     }
3395
3396     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3397         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3398         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3399         push @$devices, @$audio_devs;
3400     }
3401
3402     my $sockets = 1;
3403     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3404     $sockets = $conf->{sockets} if  $conf->{sockets};
3405
3406     my $cores = $conf->{cores} || 1;
3407
3408     my $maxcpus = $sockets * $cores;
3409
3410     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3411
3412     my $allowed_vcpus = $cpuinfo->{cpus};
3413
3414     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3415         if ($allowed_vcpus < $maxcpus);
3416
3417     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3418
3419         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3420         for (my $i = 2; $i <= $vcpus; $i++)  {
3421             my $cpustr = print_cpu_device($conf,$i);
3422             push @$cmd, '-device', $cpustr;
3423         }
3424
3425     } else {
3426
3427         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3428     }
3429     push @$cmd, '-nodefaults';
3430
3431     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3432
3433     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3434
3435     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3436
3437     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3438         push @$devices, '-device', print_vga_device(
3439             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3440         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3441         push @$cmd,  '-vnc', "unix:$socket,password";
3442     } else {
3443         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3444         push @$cmd, '-nographic';
3445     }
3446
3447     # time drift fix
3448     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3449     my $useLocaltime = $conf->{localtime};
3450
3451     if ($winversion >= 5) { # windows
3452         $useLocaltime = 1 if !defined($conf->{localtime});
3453
3454         # use time drift fix when acpi is enabled
3455         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3456             $tdf = 1 if !defined($conf->{tdf});
3457         }
3458     }
3459
3460     if ($winversion >= 6) {
3461         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3462         push @$cmd, '-no-hpet';
3463     }
3464
3465     push @$rtcFlags, 'driftfix=slew' if $tdf;
3466
3467     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3468         push @$rtcFlags, "base=$conf->{startdate}";
3469     } elsif ($useLocaltime) {
3470         push @$rtcFlags, 'base=localtime';
3471     }
3472
3473     if ($forcecpu) {
3474         push @$cmd, '-cpu', $forcecpu;
3475     } else {
3476         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3477     }
3478
3479     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3480
3481     push @$cmd, '-S' if $conf->{freeze};
3482
3483     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3484
3485     my $guest_agent = parse_guest_agent($conf);
3486
3487     if ($guest_agent->{enabled}) {
3488         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3489         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3490
3491         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3492             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3493             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3494             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3495         } elsif ($guest_agent->{type} eq 'isa') {
3496             push @$devices, '-device', "isa-serial,chardev=qga0";
3497         }
3498     }
3499
3500     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3501     if ($rng && $version_guard->(4, 1, 2)) {
3502         check_rng_source($rng->{source});
3503
3504         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3505         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3506         my $limiter_str = "";
3507         if ($max_bytes) {
3508             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3509         }
3510
3511         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3512         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3513         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3514     }
3515
3516     my $spice_port;
3517
3518     if ($qxlnum) {
3519         if ($qxlnum > 1) {
3520             if ($winversion){
3521                 for (my $i = 1; $i < $qxlnum; $i++){
3522                     push @$devices, '-device', print_vga_device(
3523                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3524                 }
3525             } else {
3526                 # assume other OS works like Linux
3527                 my ($ram, $vram) = ("134217728", "67108864");
3528                 if ($vga->{memory}) {
3529                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3530                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3531                 }
3532                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3533                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3534             }
3535         }
3536
3537         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3538
3539         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3540         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3541         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3542
3543         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3544         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3545         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3546
3547         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3548         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3549
3550         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3551         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3552         if ($spice_enhancement->{foldersharing}) {
3553             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3554             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3555         }
3556
3557         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3558         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3559             if $spice_enhancement->{videostreaming};
3560
3561         push @$devices, '-spice', "$spice_opts";
3562     }
3563
3564     # enable balloon by default, unless explicitly disabled
3565     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3566         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3567         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3568     }
3569
3570     if ($conf->{watchdog}) {
3571         my $wdopts = parse_watchdog($conf->{watchdog});
3572         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3573         my $watchdog = $wdopts->{model} || 'i6300esb';
3574         push @$devices, '-device', "$watchdog$pciaddr";
3575         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3576     }
3577
3578     my $vollist = [];
3579     my $scsicontroller = {};
3580     my $ahcicontroller = {};
3581     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3582
3583     # Add iscsi initiator name if available
3584     if (my $initiator = get_initiator_name()) {
3585         push @$devices, '-iscsi', "initiator-name=$initiator";
3586     }
3587
3588     PVE::QemuConfig->foreach_volume($conf, sub {
3589         my ($ds, $drive) = @_;
3590
3591         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3592             push @$vollist, $drive->{file};
3593         }
3594
3595         # ignore efidisk here, already added in bios/fw handling code above
3596         return if $drive->{interface} eq 'efidisk';
3597
3598         $use_virtio = 1 if $ds =~ m/^virtio/;
3599
3600         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3601
3602         if ($drive->{interface} eq 'virtio'){
3603            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3604         }
3605
3606         if ($drive->{interface} eq 'scsi') {
3607
3608             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3609
3610             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3611                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3612
3613             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3614             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3615
3616             my $iothread = '';
3617             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3618                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3619                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3620             } elsif ($drive->{iothread}) {
3621                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3622             }
3623
3624             my $queues = '';
3625             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3626                 $queues = ",num_queues=$drive->{queues}";
3627             }
3628
3629             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3630                 if !$scsicontroller->{$controller};
3631             $scsicontroller->{$controller}=1;
3632         }
3633
3634         if ($drive->{interface} eq 'sata') {
3635             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3636             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3637             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3638                 if !$ahcicontroller->{$controller};
3639             $ahcicontroller->{$controller}=1;
3640         }
3641
3642         my $pbs_conf = $pbs_backing->{$ds};
3643         my $pbs_name = undef;
3644         if ($pbs_conf) {
3645             $pbs_name = "drive-$ds-pbs";
3646             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3647         }
3648
3649         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3650         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3651
3652         push @$devices, '-drive',$drive_cmd;
3653         push @$devices, '-device', print_drivedevice_full(
3654             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3655     });
3656
3657     for (my $i = 0; $i < $MAX_NETS; $i++) {
3658         my $netname = "net$i";
3659
3660         next if !$conf->{$netname};
3661         my $d = parse_net($conf->{$netname});
3662         next if !$d;
3663
3664         $use_virtio = 1 if $d->{model} eq 'virtio';
3665
3666         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3667
3668         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3669         push @$devices, '-netdev', $netdevfull;
3670
3671         my $netdevicefull = print_netdevice_full(
3672             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3673
3674         push @$devices, '-device', $netdevicefull;
3675     }
3676
3677     if ($conf->{ivshmem}) {
3678         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3679
3680         my $bus;
3681         if ($q35) {
3682             $bus = print_pcie_addr("ivshmem");
3683         } else {
3684             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3685         }
3686
3687         my $ivshmem_name = $ivshmem->{name} // $vmid;
3688         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3689
3690         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3691         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3692             .",size=$ivshmem->{size}M";
3693     }
3694
3695     # pci.4 is nested in pci.1
3696     $bridges->{1} = 1 if $bridges->{4};
3697
3698     if (!$q35) {
3699         # add pci bridges
3700         if (min_version($machine_version, 2, 3)) {
3701            $bridges->{1} = 1;
3702            $bridges->{2} = 1;
3703         }
3704
3705         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3706
3707     }
3708
3709     for my $k (sort {$b cmp $a} keys %$bridges) {
3710         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3711
3712         my $k_name = $k;
3713         if ($k == 2 && $legacy_igd) {
3714             $k_name = "$k-igd";
3715         }
3716         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3717
3718         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3719         if ($q35) {
3720             # add after -readconfig pve-q35.cfg
3721             splice @$devices, 2, 0, '-device', $devstr;
3722         } else {
3723             unshift @$devices, '-device', $devstr if $k > 0;
3724         }
3725     }
3726
3727     if (!$kvm) {
3728         push @$machineFlags, 'accel=tcg';
3729     }
3730
3731     my $machine_type_min = $machine_type;
3732     if ($add_pve_version) {
3733         $machine_type_min =~ s/\+pve\d+$//;
3734         $machine_type_min .= "+pve$required_pve_version";
3735     }
3736     push @$machineFlags, "type=${machine_type_min}";
3737
3738     push @$cmd, @$devices;
3739     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3740     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3741     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3742
3743     if (my $vmstate = $conf->{vmstate}) {
3744         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3745         push @$vollist, $vmstate;
3746         push @$cmd, '-loadstate', $statepath;
3747         print "activating and using '$vmstate' as vmstate\n";
3748     }
3749
3750     # add custom args
3751     if ($conf->{args}) {
3752         my $aa = PVE::Tools::split_args($conf->{args});
3753         push @$cmd, @$aa;
3754     }
3755
3756     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3757 }
3758
3759 sub check_rng_source {
3760     my ($source) = @_;
3761
3762     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3763     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3764         if ! -e $source;
3765
3766     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3767     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3768         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3769         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3770         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3771             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3772             ." to the host.\n";
3773     }
3774 }
3775
3776 sub spice_port {
3777     my ($vmid) = @_;
3778
3779     my $res = mon_cmd($vmid, 'query-spice');
3780
3781     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3782 }
3783
3784 sub vm_devices_list {
3785     my ($vmid) = @_;
3786
3787     my $res = mon_cmd($vmid, 'query-pci');
3788     my $devices_to_check = [];
3789     my $devices = {};
3790     foreach my $pcibus (@$res) {
3791         push @$devices_to_check, @{$pcibus->{devices}},
3792     }
3793
3794     while (@$devices_to_check) {
3795         my $to_check = [];
3796         for my $d (@$devices_to_check) {
3797             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3798             next if !$d->{'pci_bridge'};
3799
3800             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3801             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3802         }
3803         $devices_to_check = $to_check;
3804     }
3805
3806     my $resblock = mon_cmd($vmid, 'query-block');
3807     foreach my $block (@$resblock) {
3808         if($block->{device} =~ m/^drive-(\S+)/){
3809                 $devices->{$1} = 1;
3810         }
3811     }
3812
3813     my $resmice = mon_cmd($vmid, 'query-mice');
3814     foreach my $mice (@$resmice) {
3815         if ($mice->{name} eq 'QEMU HID Tablet') {
3816             $devices->{tablet} = 1;
3817             last;
3818         }
3819     }
3820
3821     # for usb devices there is no query-usb
3822     # but we can iterate over the entries in
3823     # qom-list path=/machine/peripheral
3824     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3825     foreach my $per (@$resperipheral) {
3826         if ($per->{name} =~ m/^usb\d+$/) {
3827             $devices->{$per->{name}} = 1;
3828         }
3829     }
3830
3831     return $devices;
3832 }
3833
3834 sub vm_deviceplug {
3835     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3836
3837     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3838
3839     my $devices_list = vm_devices_list($vmid);
3840     return 1 if defined($devices_list->{$deviceid});
3841
3842     # add PCI bridge if we need it for the device
3843     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3844
3845     if ($deviceid eq 'tablet') {
3846
3847         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3848
3849     } elsif ($deviceid eq 'keyboard') {
3850
3851         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3852
3853     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3854
3855         die "usb hotplug currently not reliable\n";
3856         # since we can't reliably hot unplug all added usb devices and usb
3857         # passthrough breaks live migration we disable usb hotplugging for now
3858         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3859
3860     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3861
3862         qemu_iothread_add($vmid, $deviceid, $device);
3863
3864         qemu_driveadd($storecfg, $vmid, $device);
3865         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3866
3867         qemu_deviceadd($vmid, $devicefull);
3868         eval { qemu_deviceaddverify($vmid, $deviceid); };
3869         if (my $err = $@) {
3870             eval { qemu_drivedel($vmid, $deviceid); };
3871             warn $@ if $@;
3872             die $err;
3873         }
3874
3875     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3876
3877
3878         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3879         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3880         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3881
3882         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3883
3884         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3885             qemu_iothread_add($vmid, $deviceid, $device);
3886             $devicefull .= ",iothread=iothread-$deviceid";
3887         }
3888
3889         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3890             $devicefull .= ",num_queues=$device->{queues}";
3891         }
3892
3893         qemu_deviceadd($vmid, $devicefull);
3894         qemu_deviceaddverify($vmid, $deviceid);
3895
3896     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3897
3898         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3899         qemu_driveadd($storecfg, $vmid, $device);
3900
3901         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3902         eval { qemu_deviceadd($vmid, $devicefull); };
3903         if (my $err = $@) {
3904             eval { qemu_drivedel($vmid, $deviceid); };
3905             warn $@ if $@;
3906             die $err;
3907         }
3908
3909     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3910
3911         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3912
3913         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3914         my $use_old_bios_files = undef;
3915         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3916
3917         my $netdevicefull = print_netdevice_full(
3918             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3919         qemu_deviceadd($vmid, $netdevicefull);
3920         eval {
3921             qemu_deviceaddverify($vmid, $deviceid);
3922             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3923         };
3924         if (my $err = $@) {
3925             eval { qemu_netdevdel($vmid, $deviceid); };
3926             warn $@ if $@;
3927             die $err;
3928         }
3929
3930     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3931
3932         my $bridgeid = $2;
3933         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3934         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3935
3936         qemu_deviceadd($vmid, $devicefull);
3937         qemu_deviceaddverify($vmid, $deviceid);
3938
3939     } else {
3940         die "can't hotplug device '$deviceid'\n";
3941     }
3942
3943     return 1;
3944 }
3945
3946 # fixme: this should raise exceptions on error!
3947 sub vm_deviceunplug {
3948     my ($vmid, $conf, $deviceid) = @_;
3949
3950     my $devices_list = vm_devices_list($vmid);
3951     return 1 if !defined($devices_list->{$deviceid});
3952
3953     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3954     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3955
3956     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3957
3958         qemu_devicedel($vmid, $deviceid);
3959
3960     } elsif ($deviceid =~ m/^usb\d+$/) {
3961
3962         die "usb hotplug currently not reliable\n";
3963         # when unplugging usb devices this way, there may be remaining usb
3964         # controllers/hubs so we disable it for now
3965         #qemu_devicedel($vmid, $deviceid);
3966         #qemu_devicedelverify($vmid, $deviceid);
3967
3968     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3969
3970         qemu_devicedel($vmid, $deviceid);
3971         qemu_devicedelverify($vmid, $deviceid);
3972         qemu_drivedel($vmid, $deviceid);
3973         qemu_iothread_del($conf, $vmid, $deviceid);
3974
3975     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3976
3977         qemu_devicedel($vmid, $deviceid);
3978         qemu_devicedelverify($vmid, $deviceid);
3979         qemu_iothread_del($conf, $vmid, $deviceid);
3980
3981     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3982
3983         qemu_devicedel($vmid, $deviceid);
3984         qemu_drivedel($vmid, $deviceid);
3985         qemu_deletescsihw($conf, $vmid, $deviceid);
3986
3987     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3988
3989         qemu_devicedel($vmid, $deviceid);
3990         qemu_devicedelverify($vmid, $deviceid);
3991         qemu_netdevdel($vmid, $deviceid);
3992
3993     } else {
3994         die "can't unplug device '$deviceid'\n";
3995     }
3996
3997     return 1;
3998 }
3999
4000 sub qemu_deviceadd {
4001     my ($vmid, $devicefull) = @_;
4002
4003     $devicefull = "driver=".$devicefull;
4004     my %options =  split(/[=,]/, $devicefull);
4005
4006     mon_cmd($vmid, "device_add" , %options);
4007 }
4008
4009 sub qemu_devicedel {
4010     my ($vmid, $deviceid) = @_;
4011
4012     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4013 }
4014
4015 sub qemu_iothread_add {
4016     my($vmid, $deviceid, $device) = @_;
4017
4018     if ($device->{iothread}) {
4019         my $iothreads = vm_iothreads_list($vmid);
4020         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4021     }
4022 }
4023
4024 sub qemu_iothread_del {
4025     my($conf, $vmid, $deviceid) = @_;
4026
4027     my $confid = $deviceid;
4028     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4029         $confid = 'scsi' . $1;
4030     }
4031     my $device = parse_drive($confid, $conf->{$confid});
4032     if ($device->{iothread}) {
4033         my $iothreads = vm_iothreads_list($vmid);
4034         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4035     }
4036 }
4037
4038 sub qemu_objectadd {
4039     my($vmid, $objectid, $qomtype) = @_;
4040
4041     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4042
4043     return 1;
4044 }
4045
4046 sub qemu_objectdel {
4047     my($vmid, $objectid) = @_;
4048
4049     mon_cmd($vmid, "object-del", id => $objectid);
4050
4051     return 1;
4052 }
4053
4054 sub qemu_driveadd {
4055     my ($storecfg, $vmid, $device) = @_;
4056
4057     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4058     $drive =~ s/\\/\\\\/g;
4059     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4060
4061     # If the command succeeds qemu prints: "OK"
4062     return 1 if $ret =~ m/OK/s;
4063
4064     die "adding drive failed: $ret\n";
4065 }
4066
4067 sub qemu_drivedel {
4068     my($vmid, $deviceid) = @_;
4069
4070     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4071     $ret =~ s/^\s+//;
4072
4073     return 1 if $ret eq "";
4074
4075     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4076     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4077
4078     die "deleting drive $deviceid failed : $ret\n";
4079 }
4080
4081 sub qemu_deviceaddverify {
4082     my ($vmid, $deviceid) = @_;
4083
4084     for (my $i = 0; $i <= 5; $i++) {
4085          my $devices_list = vm_devices_list($vmid);
4086          return 1 if defined($devices_list->{$deviceid});
4087          sleep 1;
4088     }
4089
4090     die "error on hotplug device '$deviceid'\n";
4091 }
4092
4093
4094 sub qemu_devicedelverify {
4095     my ($vmid, $deviceid) = @_;
4096
4097     # need to verify that the device is correctly removed as device_del
4098     # is async and empty return is not reliable
4099
4100     for (my $i = 0; $i <= 5; $i++) {
4101          my $devices_list = vm_devices_list($vmid);
4102          return 1 if !defined($devices_list->{$deviceid});
4103          sleep 1;
4104     }
4105
4106     die "error on hot-unplugging device '$deviceid'\n";
4107 }
4108
4109 sub qemu_findorcreatescsihw {
4110     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4111
4112     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4113
4114     my $scsihwid="$controller_prefix$controller";
4115     my $devices_list = vm_devices_list($vmid);
4116
4117     if(!defined($devices_list->{$scsihwid})) {
4118         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4119     }
4120
4121     return 1;
4122 }
4123
4124 sub qemu_deletescsihw {
4125     my ($conf, $vmid, $opt) = @_;
4126
4127     my $device = parse_drive($opt, $conf->{$opt});
4128
4129     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4130         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4131         return 1;
4132     }
4133
4134     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4135
4136     my $devices_list = vm_devices_list($vmid);
4137     foreach my $opt (keys %{$devices_list}) {
4138         if (is_valid_drivename($opt)) {
4139             my $drive = parse_drive($opt, $conf->{$opt});
4140             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4141                 return 1;
4142             }
4143         }
4144     }
4145
4146     my $scsihwid="scsihw$controller";
4147
4148     vm_deviceunplug($vmid, $conf, $scsihwid);
4149
4150     return 1;
4151 }
4152
4153 sub qemu_add_pci_bridge {
4154     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4155
4156     my $bridges = {};
4157
4158     my $bridgeid;
4159
4160     print_pci_addr($device, $bridges, $arch, $machine_type);
4161
4162     while (my ($k, $v) = each %$bridges) {
4163         $bridgeid = $k;
4164     }
4165     return 1 if !defined($bridgeid) || $bridgeid < 1;
4166
4167     my $bridge = "pci.$bridgeid";
4168     my $devices_list = vm_devices_list($vmid);
4169
4170     if (!defined($devices_list->{$bridge})) {
4171         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4172     }
4173
4174     return 1;
4175 }
4176
4177 sub qemu_set_link_status {
4178     my ($vmid, $device, $up) = @_;
4179
4180     mon_cmd($vmid, "set_link", name => $device,
4181                up => $up ? JSON::true : JSON::false);
4182 }
4183
4184 sub qemu_netdevadd {
4185     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4186
4187     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4188     my %options =  split(/[=,]/, $netdev);
4189
4190     if (defined(my $vhost = $options{vhost})) {
4191         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4192     }
4193
4194     if (defined(my $queues = $options{queues})) {
4195         $options{queues} = $queues + 0;
4196     }
4197
4198     mon_cmd($vmid, "netdev_add",  %options);
4199     return 1;
4200 }
4201
4202 sub qemu_netdevdel {
4203     my ($vmid, $deviceid) = @_;
4204
4205     mon_cmd($vmid, "netdev_del", id => $deviceid);
4206 }
4207
4208 sub qemu_usb_hotplug {
4209     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4210
4211     return if !$device;
4212
4213     # remove the old one first
4214     vm_deviceunplug($vmid, $conf, $deviceid);
4215
4216     # check if xhci controller is necessary and available
4217     if ($device->{usb3}) {
4218
4219         my $devicelist = vm_devices_list($vmid);
4220
4221         if (!$devicelist->{xhci}) {
4222             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4223             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4224         }
4225     }
4226     my $d = parse_usb_device($device->{host});
4227     $d->{usb3} = $device->{usb3};
4228
4229     # add the new one
4230     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4231 }
4232
4233 sub qemu_cpu_hotplug {
4234     my ($vmid, $conf, $vcpus) = @_;
4235
4236     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4237
4238     my $sockets = 1;
4239     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4240     $sockets = $conf->{sockets} if  $conf->{sockets};
4241     my $cores = $conf->{cores} || 1;
4242     my $maxcpus = $sockets * $cores;
4243
4244     $vcpus = $maxcpus if !$vcpus;
4245
4246     die "you can't add more vcpus than maxcpus\n"
4247         if $vcpus > $maxcpus;
4248
4249     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4250
4251     if ($vcpus < $currentvcpus) {
4252
4253         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4254
4255             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4256                 qemu_devicedel($vmid, "cpu$i");
4257                 my $retry = 0;
4258                 my $currentrunningvcpus = undef;
4259                 while (1) {
4260                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4261                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4262                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4263                     $retry++;
4264                     sleep 1;
4265                 }
4266                 #update conf after each succesfull cpu unplug
4267                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4268                 PVE::QemuConfig->write_config($vmid, $conf);
4269             }
4270         } else {
4271             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4272         }
4273
4274         return;
4275     }
4276
4277     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4278     die "vcpus in running vm does not match its configuration\n"
4279         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4280
4281     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4282
4283         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4284             my $cpustr = print_cpu_device($conf, $i);
4285             qemu_deviceadd($vmid, $cpustr);
4286
4287             my $retry = 0;
4288             my $currentrunningvcpus = undef;
4289             while (1) {
4290                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4291                 last if scalar(@{$currentrunningvcpus}) == $i;
4292                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4293                 sleep 1;
4294                 $retry++;
4295             }
4296             #update conf after each succesfull cpu hotplug
4297             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4298             PVE::QemuConfig->write_config($vmid, $conf);
4299         }
4300     } else {
4301
4302         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4303             mon_cmd($vmid, "cpu-add", id => int($i));
4304         }
4305     }
4306 }
4307
4308 sub qemu_block_set_io_throttle {
4309     my ($vmid, $deviceid,
4310         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4311         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4312         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4313         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4314
4315     return if !check_running($vmid) ;
4316
4317     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4318         bps => int($bps),
4319         bps_rd => int($bps_rd),
4320         bps_wr => int($bps_wr),
4321         iops => int($iops),
4322         iops_rd => int($iops_rd),
4323         iops_wr => int($iops_wr),
4324         bps_max => int($bps_max),
4325         bps_rd_max => int($bps_rd_max),
4326         bps_wr_max => int($bps_wr_max),
4327         iops_max => int($iops_max),
4328         iops_rd_max => int($iops_rd_max),
4329         iops_wr_max => int($iops_wr_max),
4330         bps_max_length => int($bps_max_length),
4331         bps_rd_max_length => int($bps_rd_max_length),
4332         bps_wr_max_length => int($bps_wr_max_length),
4333         iops_max_length => int($iops_max_length),
4334         iops_rd_max_length => int($iops_rd_max_length),
4335         iops_wr_max_length => int($iops_wr_max_length),
4336     );
4337
4338 }
4339
4340 sub qemu_block_resize {
4341     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4342
4343     my $running = check_running($vmid);
4344
4345     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4346
4347     return if !$running;
4348
4349     my $padding = (1024 - $size % 1024) % 1024;
4350     $size = $size + $padding;
4351
4352     mon_cmd(
4353         $vmid,
4354         "block_resize",
4355         device => $deviceid,
4356         size => int($size),
4357         timeout => 60,
4358     );
4359 }
4360
4361 sub qemu_volume_snapshot {
4362     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4363
4364     my $running = check_running($vmid);
4365
4366     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4367         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4368     } else {
4369         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4370     }
4371 }
4372
4373 sub qemu_volume_snapshot_delete {
4374     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4375
4376     my $running = check_running($vmid);
4377
4378     if($running) {
4379
4380         $running = undef;
4381         my $conf = PVE::QemuConfig->load_config($vmid);
4382         PVE::QemuConfig->foreach_volume($conf, sub {
4383             my ($ds, $drive) = @_;
4384             $running = 1 if $drive->{file} eq $volid;
4385         });
4386     }
4387
4388     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4389         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4390     } else {
4391         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4392     }
4393 }
4394
4395 sub set_migration_caps {
4396     my ($vmid, $savevm) = @_;
4397
4398     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4399
4400     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4401     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4402
4403     my $cap_ref = [];
4404
4405     my $enabled_cap = {
4406         "auto-converge" => 1,
4407         "xbzrle" => 1,
4408         "x-rdma-pin-all" => 0,
4409         "zero-blocks" => 0,
4410         "compress" => 0,
4411         "dirty-bitmaps" => $dirty_bitmaps,
4412     };
4413
4414     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4415
4416     for my $supported_capability (@$supported_capabilities) {
4417         push @$cap_ref, {
4418             capability => $supported_capability->{capability},
4419             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4420         };
4421     }
4422
4423     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4424 }
4425
4426 sub foreach_volid {
4427     my ($conf, $func, @param) = @_;
4428
4429     my $volhash = {};
4430
4431     my $test_volid = sub {
4432         my ($key, $drive, $snapname) = @_;
4433
4434         my $volid = $drive->{file};
4435         return if !$volid;
4436
4437         $volhash->{$volid}->{cdrom} //= 1;
4438         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4439
4440         my $replicate = $drive->{replicate} // 1;
4441         $volhash->{$volid}->{replicate} //= 0;
4442         $volhash->{$volid}->{replicate} = 1 if $replicate;
4443
4444         $volhash->{$volid}->{shared} //= 0;
4445         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4446
4447         $volhash->{$volid}->{referenced_in_config} //= 0;
4448         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4449
4450         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4451             if defined($snapname);
4452
4453         my $size = $drive->{size};
4454         $volhash->{$volid}->{size} //= $size if $size;
4455
4456         $volhash->{$volid}->{is_vmstate} //= 0;
4457         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4458
4459         $volhash->{$volid}->{is_unused} //= 0;
4460         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4461
4462         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4463     };
4464
4465     my $include_opts = {
4466         extra_keys => ['vmstate'],
4467         include_unused => 1,
4468     };
4469
4470     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4471     foreach my $snapname (keys %{$conf->{snapshots}}) {
4472         my $snap = $conf->{snapshots}->{$snapname};
4473         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4474     }
4475
4476     foreach my $volid (keys %$volhash) {
4477         &$func($volid, $volhash->{$volid}, @param);
4478     }
4479 }
4480
4481 my $fast_plug_option = {
4482     'lock' => 1,
4483     'name' => 1,
4484     'onboot' => 1,
4485     'shares' => 1,
4486     'startup' => 1,
4487     'description' => 1,
4488     'protection' => 1,
4489     'vmstatestorage' => 1,
4490     'hookscript' => 1,
4491     'tags' => 1,
4492 };
4493
4494 # hotplug changes in [PENDING]
4495 # $selection hash can be used to only apply specified options, for
4496 # example: { cores => 1 } (only apply changed 'cores')
4497 # $errors ref is used to return error messages
4498 sub vmconfig_hotplug_pending {
4499     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4500
4501     my $defaults = load_defaults();
4502     my $arch = get_vm_arch($conf);
4503     my $machine_type = get_vm_machine($conf, undef, $arch);
4504
4505     # commit values which do not have any impact on running VM first
4506     # Note: those option cannot raise errors, we we do not care about
4507     # $selection and always apply them.
4508
4509     my $add_error = sub {
4510         my ($opt, $msg) = @_;
4511         $errors->{$opt} = "hotplug problem - $msg";
4512     };
4513
4514     my $changes = 0;
4515     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4516         if ($fast_plug_option->{$opt}) {
4517             $conf->{$opt} = $conf->{pending}->{$opt};
4518             delete $conf->{pending}->{$opt};
4519             $changes = 1;
4520         }
4521     }
4522
4523     if ($changes) {
4524         PVE::QemuConfig->write_config($vmid, $conf);
4525     }
4526
4527     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4528
4529     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4530     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4531     foreach my $opt (sort keys %$pending_delete_hash) {
4532         next if $selection && !$selection->{$opt};
4533         my $force = $pending_delete_hash->{$opt}->{force};
4534         eval {
4535             if ($opt eq 'hotplug') {
4536                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4537             } elsif ($opt eq 'tablet') {
4538                 die "skip\n" if !$hotplug_features->{usb};
4539                 if ($defaults->{tablet}) {
4540                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4541                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4542                         if $arch eq 'aarch64';
4543                 } else {
4544                     vm_deviceunplug($vmid, $conf, 'tablet');
4545                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4546                 }
4547             } elsif ($opt =~ m/^usb\d+/) {
4548                 die "skip\n";
4549                 # since we cannot reliably hot unplug usb devices we are disabling it
4550                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4551                 #vm_deviceunplug($vmid, $conf, $opt);
4552             } elsif ($opt eq 'vcpus') {
4553                 die "skip\n" if !$hotplug_features->{cpu};
4554                 qemu_cpu_hotplug($vmid, $conf, undef);
4555             } elsif ($opt eq 'balloon') {
4556                 # enable balloon device is not hotpluggable
4557                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4558                 # here we reset the ballooning value to memory
4559                 my $balloon = $conf->{memory} || $defaults->{memory};
4560                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4561             } elsif ($fast_plug_option->{$opt}) {
4562                 # do nothing
4563             } elsif ($opt =~ m/^net(\d+)$/) {
4564                 die "skip\n" if !$hotplug_features->{network};
4565                 vm_deviceunplug($vmid, $conf, $opt);
4566             } elsif (is_valid_drivename($opt)) {
4567                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4568                 vm_deviceunplug($vmid, $conf, $opt);
4569                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4570             } elsif ($opt =~ m/^memory$/) {
4571                 die "skip\n" if !$hotplug_features->{memory};
4572                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4573             } elsif ($opt eq 'cpuunits') {
4574                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4575             } elsif ($opt eq 'cpulimit') {
4576                 $cgroup->change_cpu_quota(-1, 100000);
4577             } else {
4578                 die "skip\n";
4579             }
4580         };
4581         if (my $err = $@) {
4582             &$add_error($opt, $err) if $err ne "skip\n";
4583         } else {
4584             delete $conf->{$opt};
4585             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4586         }
4587     }
4588
4589     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4590     $apply_pending_cloudinit = sub {
4591         return if $apply_pending_cloudinit_done; # once is enough
4592         $apply_pending_cloudinit_done = 1; # once is enough
4593
4594         my ($key, $value) = @_;
4595
4596         my @cloudinit_opts = keys %$confdesc_cloudinit;
4597         foreach my $opt (keys %{$conf->{pending}}) {
4598             next if !grep { $_ eq $opt } @cloudinit_opts;
4599             $conf->{$opt} = delete $conf->{pending}->{$opt};
4600         }
4601
4602         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4603         foreach my $opt (sort keys %$pending_delete_hash) {
4604             next if !grep { $_ eq $opt } @cloudinit_opts;
4605             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4606             delete $conf->{$opt};
4607         }
4608
4609         my $new_conf = { %$conf };
4610         $new_conf->{$key} = $value;
4611         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4612     };
4613
4614     foreach my $opt (keys %{$conf->{pending}}) {
4615         next if $selection && !$selection->{$opt};
4616         my $value = $conf->{pending}->{$opt};
4617         eval {
4618             if ($opt eq 'hotplug') {
4619                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4620             } elsif ($opt eq 'tablet') {
4621                 die "skip\n" if !$hotplug_features->{usb};
4622                 if ($value == 1) {
4623                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4624                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4625                         if $arch eq 'aarch64';
4626                 } elsif ($value == 0) {
4627                     vm_deviceunplug($vmid, $conf, 'tablet');
4628                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4629                 }
4630             } elsif ($opt =~ m/^usb\d+$/) {
4631                 die "skip\n";
4632                 # since we cannot reliably hot unplug usb devices we disable it for now
4633                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4634                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4635                 #die "skip\n" if !$d;
4636                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4637             } elsif ($opt eq 'vcpus') {
4638                 die "skip\n" if !$hotplug_features->{cpu};
4639                 qemu_cpu_hotplug($vmid, $conf, $value);
4640             } elsif ($opt eq 'balloon') {
4641                 # enable/disable balloning device is not hotpluggable
4642                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4643                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4644                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4645
4646                 # allow manual ballooning if shares is set to zero
4647                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4648                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4649                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4650                 }
4651             } elsif ($opt =~ m/^net(\d+)$/) {
4652                 # some changes can be done without hotplug
4653                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4654                                     $vmid, $opt, $value, $arch, $machine_type);
4655             } elsif (is_valid_drivename($opt)) {
4656                 die "skip\n" if $opt eq 'efidisk0';
4657                 # some changes can be done without hotplug
4658                 my $drive = parse_drive($opt, $value);
4659                 if (drive_is_cloudinit($drive)) {
4660                     &$apply_pending_cloudinit($opt, $value);
4661                 }
4662                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4663                                      $vmid, $opt, $value, $arch, $machine_type);
4664             } elsif ($opt =~ m/^memory$/) { #dimms
4665                 die "skip\n" if !$hotplug_features->{memory};
4666                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4667             } elsif ($opt eq 'cpuunits') {
4668                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4669             } elsif ($opt eq 'cpulimit') {
4670                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4671                 $cgroup->change_cpu_quota($cpulimit, 100000);
4672             } else {
4673                 die "skip\n";  # skip non-hot-pluggable options
4674             }
4675         };
4676         if (my $err = $@) {
4677             &$add_error($opt, $err) if $err ne "skip\n";
4678         } else {
4679             $conf->{$opt} = $value;
4680             delete $conf->{pending}->{$opt};
4681         }
4682     }
4683
4684     PVE::QemuConfig->write_config($vmid, $conf);
4685 }
4686
4687 sub try_deallocate_drive {
4688     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4689
4690     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4691         my $volid = $drive->{file};
4692         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4693             my $sid = PVE::Storage::parse_volume_id($volid);
4694             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4695
4696             # check if the disk is really unused
4697             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4698                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4699             PVE::Storage::vdisk_free($storecfg, $volid);
4700             return 1;
4701         } else {
4702             # If vm is not owner of this disk remove from config
4703             return 1;
4704         }
4705     }
4706
4707     return;
4708 }
4709
4710 sub vmconfig_delete_or_detach_drive {
4711     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4712
4713     my $drive = parse_drive($opt, $conf->{$opt});
4714
4715     my $rpcenv = PVE::RPCEnvironment::get();
4716     my $authuser = $rpcenv->get_user();
4717
4718     if ($force) {
4719         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4720         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4721     } else {
4722         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4723     }
4724 }
4725
4726
4727
4728 sub vmconfig_apply_pending {
4729     my ($vmid, $conf, $storecfg, $errors) = @_;
4730
4731     my $add_apply_error = sub {
4732         my ($opt, $msg) = @_;
4733         my $err_msg = "unable to apply pending change $opt : $msg";
4734         $errors->{$opt} = $err_msg;
4735         warn $err_msg;
4736     };
4737
4738     # cold plug
4739
4740     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4741     foreach my $opt (sort keys %$pending_delete_hash) {
4742         my $force = $pending_delete_hash->{$opt}->{force};
4743         eval {
4744             if ($opt =~ m/^unused/) {
4745                 die "internal error";
4746             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4747                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4748             }
4749         };
4750         if (my $err = $@) {
4751             $add_apply_error->($opt, $err);
4752         } else {
4753             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4754             delete $conf->{$opt};
4755         }
4756     }
4757
4758     PVE::QemuConfig->cleanup_pending($conf);
4759
4760     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4761         next if $opt eq 'delete'; # just to be sure
4762         eval {
4763             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4764                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4765             }
4766         };
4767         if (my $err = $@) {
4768             $add_apply_error->($opt, $err);
4769         } else {
4770             $conf->{$opt} = delete $conf->{pending}->{$opt};
4771         }
4772     }
4773
4774     # write all changes at once to avoid unnecessary i/o
4775     PVE::QemuConfig->write_config($vmid, $conf);
4776 }
4777
4778 sub vmconfig_update_net {
4779     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4780
4781     my $newnet = parse_net($value);
4782
4783     if ($conf->{$opt}) {
4784         my $oldnet = parse_net($conf->{$opt});
4785
4786         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4787             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4788             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4789             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4790
4791             # for non online change, we try to hot-unplug
4792             die "skip\n" if !$hotplug;
4793             vm_deviceunplug($vmid, $conf, $opt);
4794         } else {
4795
4796             die "internal error" if $opt !~ m/net(\d+)/;
4797             my $iface = "tap${vmid}i$1";
4798
4799             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4800                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4801                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4802                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4803                 PVE::Network::tap_unplug($iface);
4804
4805                 if ($have_sdn) {
4806                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4807                 } else {
4808                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4809                 }
4810             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4811                 # Rate can be applied on its own but any change above needs to
4812                 # include the rate in tap_plug since OVS resets everything.
4813                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4814             }
4815
4816             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4817                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4818             }
4819
4820             return 1;
4821         }
4822     }
4823
4824     if ($hotplug) {
4825         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4826     } else {
4827         die "skip\n";
4828     }
4829 }
4830
4831 sub vmconfig_update_disk {
4832     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4833
4834     my $drive = parse_drive($opt, $value);
4835
4836     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4837         my $media = $drive->{media} || 'disk';
4838         my $oldmedia = $old_drive->{media} || 'disk';
4839         die "unable to change media type\n" if $media ne $oldmedia;
4840
4841         if (!drive_is_cdrom($old_drive)) {
4842
4843             if ($drive->{file} ne $old_drive->{file}) {
4844
4845                 die "skip\n" if !$hotplug;
4846
4847                 # unplug and register as unused
4848                 vm_deviceunplug($vmid, $conf, $opt);
4849                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4850
4851             } else {
4852                 # update existing disk
4853
4854                 # skip non hotpluggable value
4855                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4856                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4857                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4858                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4859                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4860                     die "skip\n";
4861                 }
4862
4863                 # apply throttle
4864                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4865                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4866                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4867                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4868                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4869                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4870                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4871                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4872                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4873                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4874                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4875                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4876                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4877                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4878                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4879                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4880                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4881                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4882
4883                     qemu_block_set_io_throttle(
4884                         $vmid,"drive-$opt",
4885                         ($drive->{mbps} || 0)*1024*1024,
4886                         ($drive->{mbps_rd} || 0)*1024*1024,
4887                         ($drive->{mbps_wr} || 0)*1024*1024,
4888                          $drive->{iops} || 0,
4889                          $drive->{iops_rd} || 0,
4890                          $drive->{iops_wr} || 0,
4891                         ($drive->{mbps_max} || 0)*1024*1024,
4892                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4893                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4894                          $drive->{iops_max} || 0,
4895                          $drive->{iops_rd_max} || 0,
4896                          $drive->{iops_wr_max} || 0,
4897                          $drive->{bps_max_length} || 1,
4898                          $drive->{bps_rd_max_length} || 1,
4899                          $drive->{bps_wr_max_length} || 1,
4900                          $drive->{iops_max_length} || 1,
4901                          $drive->{iops_rd_max_length} || 1,
4902                          $drive->{iops_wr_max_length} || 1,
4903                     );
4904
4905                 }
4906
4907                 return 1;
4908             }
4909
4910         } else { # cdrom
4911
4912             if ($drive->{file} eq 'none') {
4913                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4914                 if (drive_is_cloudinit($old_drive)) {
4915                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4916                 }
4917             } else {
4918                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4919
4920                 # force eject if locked
4921                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4922
4923                 if ($path) {
4924                     mon_cmd($vmid, "blockdev-change-medium",
4925                         id => "$opt", filename => "$path");
4926                 }
4927             }
4928
4929             return 1;
4930         }
4931     }
4932
4933     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4934     # hotplug new disks
4935     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4936     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4937 }
4938
4939 # called in locked context by incoming migration
4940 sub vm_migrate_get_nbd_disks {
4941     my ($storecfg, $conf, $replicated_volumes) = @_;
4942
4943     my $local_volumes = {};
4944     PVE::QemuConfig->foreach_volume($conf, sub {
4945         my ($ds, $drive) = @_;
4946
4947         return if drive_is_cdrom($drive);
4948
4949         my $volid = $drive->{file};
4950
4951         return if !$volid;
4952
4953         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4954
4955         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4956         return if $scfg->{shared};
4957
4958         # replicated disks re-use existing state via bitmap
4959         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4960         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4961     });
4962     return $local_volumes;
4963 }
4964
4965 # called in locked context by incoming migration
4966 sub vm_migrate_alloc_nbd_disks {
4967     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4968
4969     my $format = undef;
4970
4971     my $nbd = {};
4972     foreach my $opt (sort keys %$source_volumes) {
4973         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4974
4975         if ($use_existing) {
4976             $nbd->{$opt}->{drivestr} = print_drive($drive);
4977             $nbd->{$opt}->{volid} = $volid;
4978             $nbd->{$opt}->{replicated} = 1;
4979             next;
4980         }
4981
4982         # If a remote storage is specified and the format of the original
4983         # volume is not available there, fall back to the default format.
4984         # Otherwise use the same format as the original.
4985         if (!$storagemap->{identity}) {
4986             $storeid = map_storage($storagemap, $storeid);
4987             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4988             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4989             my $fileFormat = qemu_img_format($scfg, $volname);
4990             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4991         } else {
4992             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4993             $format = qemu_img_format($scfg, $volname);
4994         }
4995
4996         my $size = $drive->{size} / 1024;
4997         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4998         my $newdrive = $drive;
4999         $newdrive->{format} = $format;
5000         $newdrive->{file} = $newvolid;
5001         my $drivestr = print_drive($newdrive);
5002         $nbd->{$opt}->{drivestr} = $drivestr;
5003         $nbd->{$opt}->{volid} = $newvolid;
5004     }
5005
5006     return $nbd;
5007 }
5008
5009 # see vm_start_nolock for parameters, additionally:
5010 # migrate_opts:
5011 #   storagemap = parsed storage map for allocating NBD disks
5012 sub vm_start {
5013     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5014
5015     return PVE::QemuConfig->lock_config($vmid, sub {
5016         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5017
5018         die "you can't start a vm if it's a template\n"
5019             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5020
5021         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5022         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5023
5024         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5025
5026         if ($has_backup_lock && $running) {
5027             # a backup is currently running, attempt to start the guest in the
5028             # existing QEMU instance
5029             return vm_resume($vmid);
5030         }
5031
5032         PVE::QemuConfig->check_lock($conf)
5033             if !($params->{skiplock} || $has_suspended_lock);
5034
5035         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5036
5037         die "VM $vmid already running\n" if $running;
5038
5039         if (my $storagemap = $migrate_opts->{storagemap}) {
5040             my $replicated = $migrate_opts->{replicated_volumes};
5041             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5042             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5043
5044             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5045                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5046             }
5047         }
5048
5049         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5050     });
5051 }
5052
5053
5054 # params:
5055 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5056 #   skiplock => 0/1, skip checking for config lock
5057 #   skiptemplate => 0/1, skip checking whether VM is template
5058 #   forcemachine => to force Qemu machine (rollback/migration)
5059 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5060 #   timeout => in seconds
5061 #   paused => start VM in paused state (backup)
5062 #   resume => resume from hibernation
5063 #   pbs-backing => {
5064 #      sata0 => {
5065 #         repository
5066 #         snapshot
5067 #         keyfile
5068 #         archive
5069 #      },
5070 #      virtio2 => ...
5071 #   }
5072 # migrate_opts:
5073 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5074 #   migratedfrom => source node
5075 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5076 #   network => CIDR of migration network
5077 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5078 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5079 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5080 sub vm_start_nolock {
5081     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5082
5083     my $statefile = $params->{statefile};
5084     my $resume = $params->{resume};
5085
5086     my $migratedfrom = $migrate_opts->{migratedfrom};
5087     my $migration_type = $migrate_opts->{type};
5088
5089     my $res = {};
5090
5091     # clean up leftover reboot request files
5092     eval { clear_reboot_request($vmid); };
5093     warn $@ if $@;
5094
5095     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5096         vmconfig_apply_pending($vmid, $conf, $storecfg);
5097         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5098     }
5099
5100     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5101
5102     my $defaults = load_defaults();
5103
5104     # set environment variable useful inside network script
5105     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5106
5107     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5108
5109     my $forcemachine = $params->{forcemachine};
5110     my $forcecpu = $params->{forcecpu};
5111     if ($resume) {
5112         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5113         $forcemachine = $conf->{runningmachine};
5114         $forcecpu = $conf->{runningcpu};
5115         print "Resuming suspended VM\n";
5116     }
5117
5118     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5119         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5120
5121     my $migration_ip;
5122     my $get_migration_ip = sub {
5123         my ($nodename) = @_;
5124
5125         return $migration_ip if defined($migration_ip);
5126
5127         my $cidr = $migrate_opts->{network};
5128
5129         if (!defined($cidr)) {
5130             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5131             $cidr = $dc_conf->{migration}->{network};
5132         }
5133
5134         if (defined($cidr)) {
5135             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5136
5137             die "could not get IP: no address configured on local " .
5138                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5139
5140             die "could not get IP: multiple addresses configured on local " .
5141                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5142
5143             $migration_ip = @$ips[0];
5144         }
5145
5146         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5147             if !defined($migration_ip);
5148
5149         return $migration_ip;
5150     };
5151
5152     my $migrate_uri;
5153     if ($statefile) {
5154         if ($statefile eq 'tcp') {
5155             my $localip = "localhost";
5156             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5157             my $nodename = nodename();
5158
5159             if (!defined($migration_type)) {
5160                 if (defined($datacenterconf->{migration}->{type})) {
5161                     $migration_type = $datacenterconf->{migration}->{type};
5162                 } else {
5163                     $migration_type = 'secure';
5164                 }
5165             }
5166
5167             if ($migration_type eq 'insecure') {
5168                 $localip = $get_migration_ip->($nodename);
5169                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5170             }
5171
5172             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5173             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5174             $migrate_uri = "tcp:${localip}:${migrate_port}";
5175             push @$cmd, '-incoming', $migrate_uri;
5176             push @$cmd, '-S';
5177
5178         } elsif ($statefile eq 'unix') {
5179             # should be default for secure migrations as a ssh TCP forward
5180             # tunnel is not deterministic reliable ready and fails regurarly
5181             # to set up in time, so use UNIX socket forwards
5182             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5183             unlink $socket_addr;
5184
5185             $migrate_uri = "unix:$socket_addr";
5186
5187             push @$cmd, '-incoming', $migrate_uri;
5188             push @$cmd, '-S';
5189
5190         } elsif (-e $statefile) {
5191             push @$cmd, '-loadstate', $statefile;
5192         } else {
5193             my $statepath = PVE::Storage::path($storecfg, $statefile);
5194             push @$vollist, $statefile;
5195             push @$cmd, '-loadstate', $statepath;
5196         }
5197     } elsif ($params->{paused}) {
5198         push @$cmd, '-S';
5199     }
5200
5201     # host pci devices
5202     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5203       my $d = parse_hostpci($conf->{"hostpci$i"});
5204       next if !$d;
5205       my $pcidevices = $d->{pciid};
5206       foreach my $pcidevice (@$pcidevices) {
5207             my $pciid = $pcidevice->{id};
5208
5209             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5210             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5211             die "no pci device info for device '$pciid'\n" if !$info;
5212
5213             if ($d->{mdev}) {
5214                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5215                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5216             } else {
5217                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5218                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5219                 die "can't reset PCI device '$pciid'\n"
5220                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5221             }
5222       }
5223     }
5224
5225     PVE::Storage::activate_volumes($storecfg, $vollist);
5226
5227     eval {
5228         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5229             outfunc => sub {}, errfunc => sub {});
5230     };
5231     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5232     # timeout should be more than enough here...
5233     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5234
5235     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5236                                               : $defaults->{cpuunits};
5237
5238     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5239     my %run_params = (
5240         timeout => $statefile ? undef : $start_timeout,
5241         umask => 0077,
5242         noerr => 1,
5243     );
5244
5245     # when migrating, prefix QEMU output so other side can pick up any
5246     # errors that might occur and show the user
5247     if ($migratedfrom) {
5248         $run_params{quiet} = 1;
5249         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5250     }
5251
5252     my %properties = (
5253         Slice => 'qemu.slice',
5254         KillMode => 'none'
5255     );
5256
5257     if (PVE::CGroup::cgroup_mode() == 2) {
5258         $properties{CPUWeight} = $cpuunits;
5259     } else {
5260         $properties{CPUShares} = $cpuunits;
5261     }
5262
5263     if (my $cpulimit = $conf->{cpulimit}) {
5264         $properties{CPUQuota} = int($cpulimit * 100);
5265     }
5266     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5267
5268     my $run_qemu = sub {
5269         PVE::Tools::run_fork sub {
5270             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5271
5272             my $exitcode = run_command($cmd, %run_params);
5273             die "QEMU exited with code $exitcode\n" if $exitcode;
5274         };
5275     };
5276
5277     if ($conf->{hugepages}) {
5278
5279         my $code = sub {
5280             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5281             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5282
5283             PVE::QemuServer::Memory::hugepages_mount();
5284             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5285
5286             eval { $run_qemu->() };
5287             if (my $err = $@) {
5288                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5289                     if !$conf->{keephugepages};
5290                 die $err;
5291             }
5292
5293             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5294                 if !$conf->{keephugepages};
5295         };
5296         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5297
5298     } else {
5299         eval { $run_qemu->() };
5300     }
5301
5302     if (my $err = $@) {
5303         # deactivate volumes if start fails
5304         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5305         die "start failed: $err";
5306     }
5307
5308     print "migration listens on $migrate_uri\n" if $migrate_uri;
5309     $res->{migrate_uri} = $migrate_uri;
5310
5311     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5312         eval { mon_cmd($vmid, "cont"); };
5313         warn $@ if $@;
5314     }
5315
5316     #start nbd server for storage migration
5317     if (my $nbd = $migrate_opts->{nbd}) {
5318         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5319
5320         my $migrate_storage_uri;
5321         # nbd_protocol_version > 0 for unix socket support
5322         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5323             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5324             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5325             $migrate_storage_uri = "nbd:unix:$socket_path";
5326         } else {
5327             my $nodename = nodename();
5328             my $localip = $get_migration_ip->($nodename);
5329             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5330             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5331
5332             mon_cmd($vmid, "nbd-server-start", addr => {
5333                 type => 'inet',
5334                 data => {
5335                     host => "${localip}",
5336                     port => "${storage_migrate_port}",
5337                 },
5338             });
5339             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5340             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5341         }
5342
5343         $res->{migrate_storage_uri} = $migrate_storage_uri;
5344
5345         foreach my $opt (sort keys %$nbd) {
5346             my $drivestr = $nbd->{$opt}->{drivestr};
5347             my $volid = $nbd->{$opt}->{volid};
5348             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5349             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5350             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5351             print "re-using replicated volume: $opt - $volid\n"
5352                 if $nbd->{$opt}->{replicated};
5353
5354             $res->{drives}->{$opt} = $nbd->{$opt};
5355             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5356         }
5357     }
5358
5359     if ($migratedfrom) {
5360         eval {
5361             set_migration_caps($vmid);
5362         };
5363         warn $@ if $@;
5364
5365         if ($spice_port) {
5366             print "spice listens on port $spice_port\n";
5367             $res->{spice_port} = $spice_port;
5368             if ($migrate_opts->{spice_ticket}) {
5369                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5370                     $migrate_opts->{spice_ticket});
5371                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5372             }
5373         }
5374
5375     } else {
5376         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5377             if !$statefile && $conf->{balloon};
5378
5379         foreach my $opt (keys %$conf) {
5380             next if $opt !~  m/^net\d+$/;
5381             my $nicconf = parse_net($conf->{$opt});
5382             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5383         }
5384     }
5385
5386     mon_cmd($vmid, 'qom-set',
5387                 path => "machine/peripheral/balloon0",
5388                 property => "guest-stats-polling-interval",
5389                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5390
5391     if ($resume) {
5392         print "Resumed VM, removing state\n";
5393         if (my $vmstate = $conf->{vmstate}) {
5394             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5395             PVE::Storage::vdisk_free($storecfg, $vmstate);
5396         }
5397         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5398         PVE::QemuConfig->write_config($vmid, $conf);
5399     }
5400
5401     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5402
5403     return $res;
5404 }
5405
5406 sub vm_commandline {
5407     my ($storecfg, $vmid, $snapname) = @_;
5408
5409     my $conf = PVE::QemuConfig->load_config($vmid);
5410     my $forcemachine;
5411     my $forcecpu;
5412
5413     if ($snapname) {
5414         my $snapshot = $conf->{snapshots}->{$snapname};
5415         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5416
5417         # check for machine or CPU overrides in snapshot
5418         $forcemachine = $snapshot->{runningmachine};
5419         $forcecpu = $snapshot->{runningcpu};
5420
5421         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5422
5423         $conf = $snapshot;
5424     }
5425
5426     my $defaults = load_defaults();
5427
5428     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5429         $forcemachine, $forcecpu);
5430
5431     return PVE::Tools::cmd2string($cmd);
5432 }
5433
5434 sub vm_reset {
5435     my ($vmid, $skiplock) = @_;
5436
5437     PVE::QemuConfig->lock_config($vmid, sub {
5438
5439         my $conf = PVE::QemuConfig->load_config($vmid);
5440
5441         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5442
5443         mon_cmd($vmid, "system_reset");
5444     });
5445 }
5446
5447 sub get_vm_volumes {
5448     my ($conf) = @_;
5449
5450     my $vollist = [];
5451     foreach_volid($conf, sub {
5452         my ($volid, $attr) = @_;
5453
5454         return if $volid =~ m|^/|;
5455
5456         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5457         return if !$sid;
5458
5459         push @$vollist, $volid;
5460     });
5461
5462     return $vollist;
5463 }
5464
5465 sub vm_stop_cleanup {
5466     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5467
5468     eval {
5469
5470         if (!$keepActive) {
5471             my $vollist = get_vm_volumes($conf);
5472             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5473         }
5474
5475         foreach my $ext (qw(mon qmp pid vnc qga)) {
5476             unlink "/var/run/qemu-server/${vmid}.$ext";
5477         }
5478
5479         if ($conf->{ivshmem}) {
5480             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5481             # just delete it for now, VMs which have this already open do not
5482             # are affected, but new VMs will get a separated one. If this
5483             # becomes an issue we either add some sort of ref-counting or just
5484             # add a "don't delete on stop" flag to the ivshmem format.
5485             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5486         }
5487
5488         foreach my $key (keys %$conf) {
5489             next if $key !~ m/^hostpci(\d+)$/;
5490             my $hostpciindex = $1;
5491             my $d = parse_hostpci($conf->{$key});
5492             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5493
5494             foreach my $pci (@{$d->{pciid}}) {
5495                 my $pciid = $pci->{id};
5496                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5497             }
5498         }
5499
5500         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5501     };
5502     warn $@ if $@; # avoid errors - just warn
5503 }
5504
5505 # call only in locked context
5506 sub _do_vm_stop {
5507     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5508
5509     my $pid = check_running($vmid, $nocheck);
5510     return if !$pid;
5511
5512     my $conf;
5513     if (!$nocheck) {
5514         $conf = PVE::QemuConfig->load_config($vmid);
5515         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5516         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5517             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5518             $timeout = $opts->{down} if $opts->{down};
5519         }
5520         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5521     }
5522
5523     eval {
5524         if ($shutdown) {
5525             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5526                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5527             } else {
5528                 mon_cmd($vmid, "system_powerdown");
5529             }
5530         } else {
5531             mon_cmd($vmid, "quit");
5532         }
5533     };
5534     my $err = $@;
5535
5536     if (!$err) {
5537         $timeout = 60 if !defined($timeout);
5538
5539         my $count = 0;
5540         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5541             $count++;
5542             sleep 1;
5543         }
5544
5545         if ($count >= $timeout) {
5546             if ($force) {
5547                 warn "VM still running - terminating now with SIGTERM\n";
5548                 kill 15, $pid;
5549             } else {
5550                 die "VM quit/powerdown failed - got timeout\n";
5551             }
5552         } else {
5553             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5554             return;
5555         }
5556     } else {
5557        if (!check_running($vmid, $nocheck)) {
5558             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5559             return;
5560        }
5561        if ($force) {
5562             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5563             kill 15, $pid;
5564         } else {
5565             die "VM quit/powerdown failed\n";
5566         }
5567     }
5568
5569     # wait again
5570     $timeout = 10;
5571
5572     my $count = 0;
5573     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5574         $count++;
5575         sleep 1;
5576     }
5577
5578     if ($count >= $timeout) {
5579         warn "VM still running - terminating now with SIGKILL\n";
5580         kill 9, $pid;
5581         sleep 1;
5582     }
5583
5584     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5585 }
5586
5587 # Note: use $nocheck to skip tests if VM configuration file exists.
5588 # We need that when migration VMs to other nodes (files already moved)
5589 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5590 sub vm_stop {
5591     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5592
5593     $force = 1 if !defined($force) && !$shutdown;
5594
5595     if ($migratedfrom){
5596         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5597         kill 15, $pid if $pid;
5598         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5599         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5600         return;
5601     }
5602
5603     PVE::QemuConfig->lock_config($vmid, sub {
5604         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5605    });
5606 }
5607
5608 sub vm_reboot {
5609     my ($vmid, $timeout) = @_;
5610
5611     PVE::QemuConfig->lock_config($vmid, sub {
5612         eval {
5613
5614             # only reboot if running, as qmeventd starts it again on a stop event
5615             return if !check_running($vmid);
5616
5617             create_reboot_request($vmid);
5618
5619             my $storecfg = PVE::Storage::config();
5620             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5621
5622         };
5623         if (my $err = $@) {
5624             # avoid that the next normal shutdown will be confused for a reboot
5625             clear_reboot_request($vmid);
5626             die $err;
5627         }
5628    });
5629 }
5630
5631 # note: if using the statestorage parameter, the caller has to check privileges
5632 sub vm_suspend {
5633     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5634
5635     my $conf;
5636     my $path;
5637     my $storecfg;
5638     my $vmstate;
5639
5640     PVE::QemuConfig->lock_config($vmid, sub {
5641
5642         $conf = PVE::QemuConfig->load_config($vmid);
5643
5644         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5645         PVE::QemuConfig->check_lock($conf)
5646             if !($skiplock || $is_backing_up);
5647
5648         die "cannot suspend to disk during backup\n"
5649             if $is_backing_up && $includestate;
5650
5651         if ($includestate) {
5652             $conf->{lock} = 'suspending';
5653             my $date = strftime("%Y-%m-%d", localtime(time()));
5654             $storecfg = PVE::Storage::config();
5655             if (!$statestorage) {
5656                 $statestorage = find_vmstate_storage($conf, $storecfg);
5657                 # check permissions for the storage
5658                 my $rpcenv = PVE::RPCEnvironment::get();
5659                 if ($rpcenv->{type} ne 'cli') {
5660                     my $authuser = $rpcenv->get_user();
5661                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5662                 }
5663             }
5664
5665
5666             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5667                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5668             $path = PVE::Storage::path($storecfg, $vmstate);
5669             PVE::QemuConfig->write_config($vmid, $conf);
5670         } else {
5671             mon_cmd($vmid, "stop");
5672         }
5673     });
5674
5675     if ($includestate) {
5676         # save vm state
5677         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5678
5679         eval {
5680             set_migration_caps($vmid, 1);
5681             mon_cmd($vmid, "savevm-start", statefile => $path);
5682             for(;;) {
5683                 my $state = mon_cmd($vmid, "query-savevm");
5684                 if (!$state->{status}) {
5685                     die "savevm not active\n";
5686                 } elsif ($state->{status} eq 'active') {
5687                     sleep(1);
5688                     next;
5689                 } elsif ($state->{status} eq 'completed') {
5690                     print "State saved, quitting\n";
5691                     last;
5692                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5693                     die "query-savevm failed with error '$state->{error}'\n"
5694                 } else {
5695                     die "query-savevm returned status '$state->{status}'\n";
5696                 }
5697             }
5698         };
5699         my $err = $@;
5700
5701         PVE::QemuConfig->lock_config($vmid, sub {
5702             $conf = PVE::QemuConfig->load_config($vmid);
5703             if ($err) {
5704                 # cleanup, but leave suspending lock, to indicate something went wrong
5705                 eval {
5706                     mon_cmd($vmid, "savevm-end");
5707                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5708                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5709                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5710                     PVE::QemuConfig->write_config($vmid, $conf);
5711                 };
5712                 warn $@ if $@;
5713                 die $err;
5714             }
5715
5716             die "lock changed unexpectedly\n"
5717                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5718
5719             mon_cmd($vmid, "quit");
5720             $conf->{lock} = 'suspended';
5721             PVE::QemuConfig->write_config($vmid, $conf);
5722         });
5723     }
5724 }
5725
5726 sub vm_resume {
5727     my ($vmid, $skiplock, $nocheck) = @_;
5728
5729     PVE::QemuConfig->lock_config($vmid, sub {
5730         my $res = mon_cmd($vmid, 'query-status');
5731         my $resume_cmd = 'cont';
5732         my $reset = 0;
5733
5734         if ($res->{status}) {
5735             return if $res->{status} eq 'running'; # job done, go home
5736             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5737             $reset = 1 if $res->{status} eq 'shutdown';
5738         }
5739
5740         if (!$nocheck) {
5741
5742             my $conf = PVE::QemuConfig->load_config($vmid);
5743
5744             PVE::QemuConfig->check_lock($conf)
5745                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5746         }
5747
5748         if ($reset) {
5749             # required if a VM shuts down during a backup and we get a resume
5750             # request before the backup finishes for example
5751             mon_cmd($vmid, "system_reset");
5752         }
5753         mon_cmd($vmid, $resume_cmd);
5754     });
5755 }
5756
5757 sub vm_sendkey {
5758     my ($vmid, $skiplock, $key) = @_;
5759
5760     PVE::QemuConfig->lock_config($vmid, sub {
5761
5762         my $conf = PVE::QemuConfig->load_config($vmid);
5763
5764         # there is no qmp command, so we use the human monitor command
5765         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5766         die $res if $res ne '';
5767     });
5768 }
5769
5770 # vzdump restore implementaion
5771
5772 sub tar_archive_read_firstfile {
5773     my $archive = shift;
5774
5775     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5776
5777     # try to detect archive type first
5778     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5779         die "unable to open file '$archive'\n";
5780     my $firstfile = <$fh>;
5781     kill 15, $pid;
5782     close $fh;
5783
5784     die "ERROR: archive contaions no data\n" if !$firstfile;
5785     chomp $firstfile;
5786
5787     return $firstfile;
5788 }
5789
5790 sub tar_restore_cleanup {
5791     my ($storecfg, $statfile) = @_;
5792
5793     print STDERR "starting cleanup\n";
5794
5795     if (my $fd = IO::File->new($statfile, "r")) {
5796         while (defined(my $line = <$fd>)) {
5797             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5798                 my $volid = $2;
5799                 eval {
5800                     if ($volid =~ m|^/|) {
5801                         unlink $volid || die 'unlink failed\n';
5802                     } else {
5803                         PVE::Storage::vdisk_free($storecfg, $volid);
5804                     }
5805                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5806                 };
5807                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5808             } else {
5809                 print STDERR "unable to parse line in statfile - $line";
5810             }
5811         }
5812         $fd->close();
5813     }
5814 }
5815
5816 sub restore_file_archive {
5817     my ($archive, $vmid, $user, $opts) = @_;
5818
5819     return restore_vma_archive($archive, $vmid, $user, $opts)
5820         if $archive eq '-';
5821
5822     my $info = PVE::Storage::archive_info($archive);
5823     my $format = $opts->{format} // $info->{format};
5824     my $comp = $info->{compression};
5825
5826     # try to detect archive format
5827     if ($format eq 'tar') {
5828         return restore_tar_archive($archive, $vmid, $user, $opts);
5829     } else {
5830         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5831     }
5832 }
5833
5834 # hepler to remove disks that will not be used after restore
5835 my $restore_cleanup_oldconf = sub {
5836     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5837
5838     PVE::QemuConfig->foreach_volume($oldconf, sub {
5839         my ($ds, $drive) = @_;
5840
5841         return if drive_is_cdrom($drive, 1);
5842
5843         my $volid = $drive->{file};
5844         return if !$volid || $volid =~ m|^/|;
5845
5846         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5847         return if !$path || !$owner || ($owner != $vmid);
5848
5849         # Note: only delete disk we want to restore
5850         # other volumes will become unused
5851         if ($virtdev_hash->{$ds}) {
5852             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5853             if (my $err = $@) {
5854                 warn $err;
5855             }
5856         }
5857     });
5858
5859     # delete vmstate files, after the restore we have no snapshots anymore
5860     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5861         my $snap = $oldconf->{snapshots}->{$snapname};
5862         if ($snap->{vmstate}) {
5863             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5864             if (my $err = $@) {
5865                 warn $err;
5866             }
5867         }
5868     }
5869 };
5870
5871 # Helper to parse vzdump backup device hints
5872 #
5873 # $rpcenv: Environment, used to ckeck storage permissions
5874 # $user: User ID, to check storage permissions
5875 # $storecfg: Storage configuration
5876 # $fh: the file handle for reading the configuration
5877 # $devinfo: should contain device sizes for all backu-up'ed devices
5878 # $options: backup options (pool, default storage)
5879 #
5880 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5881 my $parse_backup_hints = sub {
5882     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5883
5884     my $virtdev_hash = {};
5885
5886     while (defined(my $line = <$fh>)) {
5887         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5888             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5889             die "archive does not contain data for drive '$virtdev'\n"
5890                 if !$devinfo->{$devname};
5891
5892             if (defined($options->{storage})) {
5893                 $storeid = $options->{storage} || 'local';
5894             } elsif (!$storeid) {
5895                 $storeid = 'local';
5896             }
5897             $format = 'raw' if !$format;
5898             $devinfo->{$devname}->{devname} = $devname;
5899             $devinfo->{$devname}->{virtdev} = $virtdev;
5900             $devinfo->{$devname}->{format} = $format;
5901             $devinfo->{$devname}->{storeid} = $storeid;
5902
5903             # check permission on storage
5904             my $pool = $options->{pool}; # todo: do we need that?
5905             if ($user ne 'root@pam') {
5906                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5907             }
5908
5909             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5910         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5911             my $virtdev = $1;
5912             my $drive = parse_drive($virtdev, $2);
5913             if (drive_is_cloudinit($drive)) {
5914                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5915                 $storeid = $options->{storage} if defined ($options->{storage});
5916                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5917                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5918
5919                 $virtdev_hash->{$virtdev} = {
5920                     format => $format,
5921                     storeid => $storeid,
5922                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5923                     is_cloudinit => 1,
5924                 };
5925             }
5926         }
5927     }
5928
5929     return $virtdev_hash;
5930 };
5931
5932 # Helper to allocate and activate all volumes required for a restore
5933 #
5934 # $storecfg: Storage configuration
5935 # $virtdev_hash: as returned by parse_backup_hints()
5936 #
5937 # Returns: { $virtdev => $volid }
5938 my $restore_allocate_devices = sub {
5939     my ($storecfg, $virtdev_hash, $vmid) = @_;
5940
5941     my $map = {};
5942     foreach my $virtdev (sort keys %$virtdev_hash) {
5943         my $d = $virtdev_hash->{$virtdev};
5944         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5945         my $storeid = $d->{storeid};
5946         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5947
5948         # test if requested format is supported
5949         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5950         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5951         $d->{format} = $defFormat if !$supported;
5952
5953         my $name;
5954         if ($d->{is_cloudinit}) {
5955             $name = "vm-$vmid-cloudinit";
5956             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5957             if ($scfg->{path}) {
5958                 $name .= ".$d->{format}";
5959             }
5960         }
5961
5962         my $volid = PVE::Storage::vdisk_alloc(
5963             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5964
5965         print STDERR "new volume ID is '$volid'\n";
5966         $d->{volid} = $volid;
5967
5968         PVE::Storage::activate_volumes($storecfg, [$volid]);
5969
5970         $map->{$virtdev} = $volid;
5971     }
5972
5973     return $map;
5974 };
5975
5976 sub restore_update_config_line {
5977     my ($cookie, $map, $line, $unique) = @_;
5978
5979     return '' if $line =~ m/^\#qmdump\#/;
5980     return '' if $line =~ m/^\#vzdump\#/;
5981     return '' if $line =~ m/^lock:/;
5982     return '' if $line =~ m/^unused\d+:/;
5983     return '' if $line =~ m/^parent:/;
5984
5985     my $res = '';
5986
5987     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5988     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5989         # try to convert old 1.X settings
5990         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5991         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5992             my ($model, $macaddr) = split(/\=/, $devconfig);
5993             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5994             my $net = {
5995                 model => $model,
5996                 bridge => "vmbr$ind",
5997                 macaddr => $macaddr,
5998             };
5999             my $netstr = print_net($net);
6000
6001             $res .= "net$cookie->{netcount}: $netstr\n";
6002             $cookie->{netcount}++;
6003         }
6004     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6005         my ($id, $netstr) = ($1, $2);
6006         my $net = parse_net($netstr);
6007         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6008         $netstr = print_net($net);
6009         $res .= "$id: $netstr\n";
6010     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6011         my $virtdev = $1;
6012         my $value = $3;
6013         my $di = parse_drive($virtdev, $value);
6014         if (defined($di->{backup}) && !$di->{backup}) {
6015             $res .= "#$line";
6016         } elsif ($map->{$virtdev}) {
6017             delete $di->{format}; # format can change on restore
6018             $di->{file} = $map->{$virtdev};
6019             $value = print_drive($di);
6020             $res .= "$virtdev: $value\n";
6021         } else {
6022             $res .= $line;
6023         }
6024     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6025         my $vmgenid = $1;
6026         if ($vmgenid ne '0') {
6027             # always generate a new vmgenid if there was a valid one setup
6028             $vmgenid = generate_uuid();
6029         }
6030         $res .= "vmgenid: $vmgenid\n";
6031     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6032         my ($uuid, $uuid_str);
6033         UUID::generate($uuid);
6034         UUID::unparse($uuid, $uuid_str);
6035         my $smbios1 = parse_smbios1($2);
6036         $smbios1->{uuid} = $uuid_str;
6037         $res .= $1.print_smbios1($smbios1)."\n";
6038     } else {
6039         $res .= $line;
6040     }
6041
6042     return $res;
6043 }
6044
6045 my $restore_deactivate_volumes = sub {
6046     my ($storecfg, $devinfo) = @_;
6047
6048     my $vollist = [];
6049     foreach my $devname (keys %$devinfo) {
6050         my $volid = $devinfo->{$devname}->{volid};
6051         push @$vollist, $volid if $volid;
6052     }
6053
6054     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6055 };
6056
6057 my $restore_destroy_volumes = sub {
6058     my ($storecfg, $devinfo) = @_;
6059
6060     foreach my $devname (keys %$devinfo) {
6061         my $volid = $devinfo->{$devname}->{volid};
6062         next if !$volid;
6063         eval {
6064             if ($volid =~ m|^/|) {
6065                 unlink $volid || die 'unlink failed\n';
6066             } else {
6067                 PVE::Storage::vdisk_free($storecfg, $volid);
6068             }
6069             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6070         };
6071         print STDERR "unable to cleanup '$volid' - $@" if $@;
6072     }
6073 };
6074
6075 # FIXME For PVE 7.0, remove $content_type and always use 'images'
6076 sub scan_volids {
6077     my ($cfg, $vmid, $content_type) = @_;
6078
6079     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, $content_type);
6080
6081     my $volid_hash = {};
6082     foreach my $storeid (keys %$info) {
6083         foreach my $item (@{$info->{$storeid}}) {
6084             next if !($item->{volid} && $item->{size});
6085             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6086             $volid_hash->{$item->{volid}} = $item;
6087         }
6088     }
6089
6090     return $volid_hash;
6091 }
6092
6093 sub update_disk_config {
6094     my ($vmid, $conf, $volid_hash) = @_;
6095
6096     my $changes;
6097     my $prefix = "VM $vmid";
6098
6099     # used and unused disks
6100     my $referenced = {};
6101
6102     # Note: it is allowed to define multiple storages with same path (alias), so
6103     # we need to check both 'volid' and real 'path' (two different volid can point
6104     # to the same path).
6105
6106     my $referencedpath = {};
6107
6108     # update size info
6109     PVE::QemuConfig->foreach_volume($conf, sub {
6110         my ($opt, $drive) = @_;
6111
6112         my $volid = $drive->{file};
6113         return if !$volid;
6114         my $volume = $volid_hash->{$volid};
6115
6116         # mark volid as "in-use" for next step
6117         $referenced->{$volid} = 1;
6118         if ($volume && (my $path = $volume->{path})) {
6119             $referencedpath->{$path} = 1;
6120         }
6121
6122         return if drive_is_cdrom($drive);
6123         return if !$volume;
6124
6125         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6126         if (defined($updated)) {
6127             $changes = 1;
6128             $conf->{$opt} = print_drive($updated);
6129             print "$prefix ($opt): $msg\n";
6130         }
6131     });
6132
6133     # remove 'unusedX' entry if volume is used
6134     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6135         my ($opt, $drive) = @_;
6136
6137         my $volid = $drive->{file};
6138         return if !$volid;
6139
6140         my $path;
6141         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6142         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6143             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6144             $changes = 1;
6145             delete $conf->{$opt};
6146         }
6147
6148         $referenced->{$volid} = 1;
6149         $referencedpath->{$path} = 1 if $path;
6150     });
6151
6152     foreach my $volid (sort keys %$volid_hash) {
6153         next if $volid =~ m/vm-$vmid-state-/;
6154         next if $referenced->{$volid};
6155         my $path = $volid_hash->{$volid}->{path};
6156         next if !$path; # just to be sure
6157         next if $referencedpath->{$path};
6158         $changes = 1;
6159         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6160         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6161         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6162     }
6163
6164     return $changes;
6165 }
6166
6167 sub rescan {
6168     my ($vmid, $nolock, $dryrun) = @_;
6169
6170     my $cfg = PVE::Storage::config();
6171
6172     print "rescan volumes...\n";
6173     my $volid_hash = scan_volids($cfg, $vmid, 'images');
6174
6175     my $updatefn =  sub {
6176         my ($vmid) = @_;
6177
6178         my $conf = PVE::QemuConfig->load_config($vmid);
6179
6180         PVE::QemuConfig->check_lock($conf);
6181
6182         my $vm_volids = {};
6183         foreach my $volid (keys %$volid_hash) {
6184             my $info = $volid_hash->{$volid};
6185             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6186         }
6187
6188         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6189
6190         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6191     };
6192
6193     if (defined($vmid)) {
6194         if ($nolock) {
6195             &$updatefn($vmid);
6196         } else {
6197             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6198         }
6199     } else {
6200         my $vmlist = config_list();
6201         foreach my $vmid (keys %$vmlist) {
6202             if ($nolock) {
6203                 &$updatefn($vmid);
6204             } else {
6205                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6206             }
6207         }
6208     }
6209 }
6210
6211 sub restore_proxmox_backup_archive {
6212     my ($archive, $vmid, $user, $options) = @_;
6213
6214     my $storecfg = PVE::Storage::config();
6215
6216     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6217     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6218
6219     my $fingerprint = $scfg->{fingerprint};
6220     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6221
6222     my $repo = PVE::PBSClient::get_repository($scfg);
6223
6224     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6225     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6226     local $ENV{PBS_PASSWORD} = $password;
6227     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6228
6229     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6230         PVE::Storage::parse_volname($storecfg, $archive);
6231
6232     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6233
6234     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6235
6236     my $tmpdir = "/var/tmp/vzdumptmp$$";
6237     rmtree $tmpdir;
6238     mkpath $tmpdir;
6239
6240     my $conffile = PVE::QemuConfig->config_file($vmid);
6241      # disable interrupts (always do cleanups)
6242     local $SIG{INT} =
6243         local $SIG{TERM} =
6244         local $SIG{QUIT} =
6245         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6246
6247     # Note: $oldconf is undef if VM does not exists
6248     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6249     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6250     my $new_conf_raw = '';
6251
6252     my $rpcenv = PVE::RPCEnvironment::get();
6253     my $devinfo = {};
6254
6255     eval {
6256         # enable interrupts
6257         local $SIG{INT} =
6258             local $SIG{TERM} =
6259             local $SIG{QUIT} =
6260             local $SIG{HUP} =
6261             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6262
6263         my $cfgfn = "$tmpdir/qemu-server.conf";
6264         my $firewall_config_fn = "$tmpdir/fw.conf";
6265         my $index_fn = "$tmpdir/index.json";
6266
6267         my $cmd = "restore";
6268
6269         my $param = [$pbs_backup_name, "index.json", $index_fn];
6270         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6271         my $index = PVE::Tools::file_get_contents($index_fn);
6272         $index = decode_json($index);
6273
6274         # print Dumper($index);
6275         foreach my $info (@{$index->{files}}) {
6276             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6277                 my $devname = $1;
6278                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6279                     $devinfo->{$devname}->{size} = $1;
6280                 } else {
6281                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6282                 }
6283             }
6284         }
6285
6286         my $is_qemu_server_backup = scalar(
6287             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6288         );
6289         if (!$is_qemu_server_backup) {
6290             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6291         }
6292         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6293
6294         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6295         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6296
6297         if ($has_firewall_config) {
6298             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6299             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6300
6301             my $pve_firewall_dir = '/etc/pve/firewall';
6302             mkdir $pve_firewall_dir; # make sure the dir exists
6303             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6304         }
6305
6306         my $fh = IO::File->new($cfgfn, "r") ||
6307             die "unable to read qemu-server.conf - $!\n";
6308
6309         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6310
6311         # fixme: rate limit?
6312
6313         # create empty/temp config
6314         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6315
6316         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6317
6318         # allocate volumes
6319         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6320
6321         if (!$options->{live}) {
6322             foreach my $virtdev (sort keys %$virtdev_hash) {
6323                 my $d = $virtdev_hash->{$virtdev};
6324                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6325
6326                 my $volid = $d->{volid};
6327
6328                 my $path = PVE::Storage::path($storecfg, $volid);
6329
6330                 my $pbs_restore_cmd = [
6331                     '/usr/bin/pbs-restore',
6332                     '--repository', $repo,
6333                     $pbs_backup_name,
6334                     "$d->{devname}.img.fidx",
6335                     $path,
6336                     '--verbose',
6337                     ];
6338
6339                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6340                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6341
6342                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6343                     push @$pbs_restore_cmd, '--skip-zero';
6344                 }
6345
6346                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6347                 print "restore proxmox backup image: $dbg_cmdstring\n";
6348                 run_command($pbs_restore_cmd);
6349             }
6350         }
6351
6352         $fh->seek(0, 0) || die "seek failed - $!\n";
6353
6354         my $cookie = { netcount => 0 };
6355         while (defined(my $line = <$fh>)) {
6356             $new_conf_raw .= restore_update_config_line(
6357                 $cookie,
6358                 $map,
6359                 $line,
6360                 $options->{unique},
6361             );
6362         }
6363
6364         $fh->close();
6365     };
6366     my $err = $@;
6367
6368     if ($err || !$options->{live}) {
6369         $restore_deactivate_volumes->($storecfg, $devinfo);
6370     }
6371
6372     rmtree $tmpdir;
6373
6374     if ($err) {
6375         $restore_destroy_volumes->($storecfg, $devinfo);
6376         die $err;
6377     }
6378
6379     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6380
6381     PVE::Cluster::cfs_update(); # make sure we read new file
6382
6383     eval { rescan($vmid, 1); };
6384     warn $@ if $@;
6385
6386     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6387
6388     if ($options->{live}) {
6389         # enable interrupts
6390         local $SIG{INT} =
6391             local $SIG{TERM} =
6392             local $SIG{QUIT} =
6393             local $SIG{HUP} =
6394             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6395
6396         my $conf = PVE::QemuConfig->load_config($vmid);
6397         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6398
6399         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6400     }
6401 }
6402
6403 sub pbs_live_restore {
6404     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6405
6406     print "Starting VM for live-restore\n";
6407
6408     my $pbs_backing = {};
6409     for my $ds (keys %$restored_disks) {
6410         $ds =~ m/^drive-(.*)$/;
6411         $pbs_backing->{$1} = {
6412             repository => $repo,
6413             snapshot => $snap,
6414             archive => "$ds.img.fidx",
6415         };
6416         $pbs_backing->{$1}->{keyfile} = $keyfile if -e $keyfile;
6417     }
6418
6419     my $drives_streamed = 0;
6420     eval {
6421         # make sure HA doesn't interrupt our restore by stopping the VM
6422         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6423             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6424         }
6425
6426         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6427         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6428         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6429
6430         my $qmeventd_fd = register_qmeventd_handle($vmid);
6431
6432         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6433         # this will effectively collapse the backing image chain consisting of
6434         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6435         # removes itself once all backing images vanish with 'auto-remove=on')
6436         my $jobs = {};
6437         for my $ds (sort keys %$restored_disks) {
6438             my $job_id = "restore-$ds";
6439             mon_cmd($vmid, 'block-stream',
6440                 'job-id' => $job_id,
6441                 device => "$ds",
6442             );
6443             $jobs->{$job_id} = {};
6444         }
6445
6446         mon_cmd($vmid, 'cont');
6447         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6448
6449         print "restore-drive jobs finished successfully, removing all tracking block devices"
6450             ." to disconnect from Proxmox Backup Server\n";
6451
6452         for my $ds (sort keys %$restored_disks) {
6453             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6454         }
6455
6456         close($qmeventd_fd);
6457     };
6458
6459     my $err = $@;
6460
6461     if ($err) {
6462         warn "An error occured during live-restore: $err\n";
6463         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6464         die "live-restore failed\n";
6465     }
6466 }
6467
6468 sub restore_vma_archive {
6469     my ($archive, $vmid, $user, $opts, $comp) = @_;
6470
6471     my $readfrom = $archive;
6472
6473     my $cfg = PVE::Storage::config();
6474     my $commands = [];
6475     my $bwlimit = $opts->{bwlimit};
6476
6477     my $dbg_cmdstring = '';
6478     my $add_pipe = sub {
6479         my ($cmd) = @_;
6480         push @$commands, $cmd;
6481         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6482         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6483         $readfrom = '-';
6484     };
6485
6486     my $input = undef;
6487     if ($archive eq '-') {
6488         $input = '<&STDIN';
6489     } else {
6490         # If we use a backup from a PVE defined storage we also consider that
6491         # storage's rate limit:
6492         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6493         if (defined($volid)) {
6494             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6495             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6496             if ($readlimit) {
6497                 print STDERR "applying read rate limit: $readlimit\n";
6498                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6499                 $add_pipe->($cstream);
6500             }
6501         }
6502     }
6503
6504     if ($comp) {
6505         my $info = PVE::Storage::decompressor_info('vma', $comp);
6506         my $cmd = $info->{decompressor};
6507         push @$cmd, $readfrom;
6508         $add_pipe->($cmd);
6509     }
6510
6511     my $tmpdir = "/var/tmp/vzdumptmp$$";
6512     rmtree $tmpdir;
6513
6514     # disable interrupts (always do cleanups)
6515     local $SIG{INT} =
6516         local $SIG{TERM} =
6517         local $SIG{QUIT} =
6518         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6519
6520     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6521     POSIX::mkfifo($mapfifo, 0600);
6522     my $fifofh;
6523     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6524
6525     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6526
6527     my $oldtimeout;
6528     my $timeout = 5;
6529
6530     my $devinfo = {};
6531
6532     my $rpcenv = PVE::RPCEnvironment::get();
6533
6534     my $conffile = PVE::QemuConfig->config_file($vmid);
6535
6536     # Note: $oldconf is undef if VM does not exist
6537     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6538     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6539     my $new_conf_raw = '';
6540
6541     my %storage_limits;
6542
6543     my $print_devmap = sub {
6544         my $cfgfn = "$tmpdir/qemu-server.conf";
6545
6546         # we can read the config - that is already extracted
6547         my $fh = IO::File->new($cfgfn, "r") ||
6548             die "unable to read qemu-server.conf - $!\n";
6549
6550         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6551         if (-f $fwcfgfn) {
6552             my $pve_firewall_dir = '/etc/pve/firewall';
6553             mkdir $pve_firewall_dir; # make sure the dir exists
6554             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6555         }
6556
6557         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6558
6559         foreach my $info (values %{$virtdev_hash}) {
6560             my $storeid = $info->{storeid};
6561             next if defined($storage_limits{$storeid});
6562
6563             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6564             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6565             $storage_limits{$storeid} = $limit * 1024;
6566         }
6567
6568         foreach my $devname (keys %$devinfo) {
6569             die "found no device mapping information for device '$devname'\n"
6570                 if !$devinfo->{$devname}->{virtdev};
6571         }
6572
6573         # create empty/temp config
6574         if ($oldconf) {
6575             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6576             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6577         }
6578
6579         # allocate volumes
6580         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6581
6582         # print restore information to $fifofh
6583         foreach my $virtdev (sort keys %$virtdev_hash) {
6584             my $d = $virtdev_hash->{$virtdev};
6585             next if $d->{is_cloudinit}; # no need to restore cloudinit
6586
6587             my $storeid = $d->{storeid};
6588             my $volid = $d->{volid};
6589
6590             my $map_opts = '';
6591             if (my $limit = $storage_limits{$storeid}) {
6592                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6593             }
6594
6595             my $write_zeros = 1;
6596             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6597                 $write_zeros = 0;
6598             }
6599
6600             my $path = PVE::Storage::path($cfg, $volid);
6601
6602             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6603
6604             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6605         }
6606
6607         $fh->seek(0, 0) || die "seek failed - $!\n";
6608
6609         my $cookie = { netcount => 0 };
6610         while (defined(my $line = <$fh>)) {
6611             $new_conf_raw .= restore_update_config_line(
6612                 $cookie,
6613                 $map,
6614                 $line,
6615                 $opts->{unique},
6616             );
6617         }
6618
6619         $fh->close();
6620     };
6621
6622     eval {
6623         # enable interrupts
6624         local $SIG{INT} =
6625             local $SIG{TERM} =
6626             local $SIG{QUIT} =
6627             local $SIG{HUP} =
6628             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6629         local $SIG{ALRM} = sub { die "got timeout\n"; };
6630
6631         $oldtimeout = alarm($timeout);
6632
6633         my $parser = sub {
6634             my $line = shift;
6635
6636             print "$line\n";
6637
6638             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6639                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6640                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6641             } elsif ($line =~ m/^CTIME: /) {
6642                 # we correctly received the vma config, so we can disable
6643                 # the timeout now for disk allocation (set to 10 minutes, so
6644                 # that we always timeout if something goes wrong)
6645                 alarm(600);
6646                 &$print_devmap();
6647                 print $fifofh "done\n";
6648                 my $tmp = $oldtimeout || 0;
6649                 $oldtimeout = undef;
6650                 alarm($tmp);
6651                 close($fifofh);
6652                 $fifofh = undef;
6653             }
6654         };
6655
6656         print "restore vma archive: $dbg_cmdstring\n";
6657         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6658     };
6659     my $err = $@;
6660
6661     alarm($oldtimeout) if $oldtimeout;
6662
6663     $restore_deactivate_volumes->($cfg, $devinfo);
6664
6665     close($fifofh) if $fifofh;
6666     unlink $mapfifo;
6667     rmtree $tmpdir;
6668
6669     if ($err) {
6670         $restore_destroy_volumes->($cfg, $devinfo);
6671         die $err;
6672     }
6673
6674     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6675
6676     PVE::Cluster::cfs_update(); # make sure we read new file
6677
6678     eval { rescan($vmid, 1); };
6679     warn $@ if $@;
6680
6681     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6682 }
6683
6684 sub restore_tar_archive {
6685     my ($archive, $vmid, $user, $opts) = @_;
6686
6687     if ($archive ne '-') {
6688         my $firstfile = tar_archive_read_firstfile($archive);
6689         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6690             if $firstfile ne 'qemu-server.conf';
6691     }
6692
6693     my $storecfg = PVE::Storage::config();
6694
6695     # avoid zombie disks when restoring over an existing VM -> cleanup first
6696     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6697     # skiplock=1 because qmrestore has set the 'create' lock itself already
6698     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6699     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6700
6701     my $tocmd = "/usr/lib/qemu-server/qmextract";
6702
6703     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6704     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6705     $tocmd .= ' --prealloc' if $opts->{prealloc};
6706     $tocmd .= ' --info' if $opts->{info};
6707
6708     # tar option "xf" does not autodetect compression when read from STDIN,
6709     # so we pipe to zcat
6710     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6711         PVE::Tools::shellquote("--to-command=$tocmd");
6712
6713     my $tmpdir = "/var/tmp/vzdumptmp$$";
6714     mkpath $tmpdir;
6715
6716     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6717     local $ENV{VZDUMP_VMID} = $vmid;
6718     local $ENV{VZDUMP_USER} = $user;
6719
6720     my $conffile = PVE::QemuConfig->config_file($vmid);
6721     my $new_conf_raw = '';
6722
6723     # disable interrupts (always do cleanups)
6724     local $SIG{INT} =
6725         local $SIG{TERM} =
6726         local $SIG{QUIT} =
6727         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6728
6729     eval {
6730         # enable interrupts
6731         local $SIG{INT} =
6732             local $SIG{TERM} =
6733             local $SIG{QUIT} =
6734             local $SIG{HUP} =
6735             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6736
6737         if ($archive eq '-') {
6738             print "extracting archive from STDIN\n";
6739             run_command($cmd, input => "<&STDIN");
6740         } else {
6741             print "extracting archive '$archive'\n";
6742             run_command($cmd);
6743         }
6744
6745         return if $opts->{info};
6746
6747         # read new mapping
6748         my $map = {};
6749         my $statfile = "$tmpdir/qmrestore.stat";
6750         if (my $fd = IO::File->new($statfile, "r")) {
6751             while (defined (my $line = <$fd>)) {
6752                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6753                     $map->{$1} = $2 if $1;
6754                 } else {
6755                     print STDERR "unable to parse line in statfile - $line\n";
6756                 }
6757             }
6758             $fd->close();
6759         }
6760
6761         my $confsrc = "$tmpdir/qemu-server.conf";
6762
6763         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6764
6765         my $cookie = { netcount => 0 };
6766         while (defined (my $line = <$srcfd>)) {
6767             $new_conf_raw .= restore_update_config_line(
6768                 $cookie,
6769                 $map,
6770                 $line,
6771                 $opts->{unique},
6772             );
6773         }
6774
6775         $srcfd->close();
6776     };
6777     if (my $err = $@) {
6778         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6779         die $err;
6780     }
6781
6782     rmtree $tmpdir;
6783
6784     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6785
6786     PVE::Cluster::cfs_update(); # make sure we read new file
6787
6788     eval { rescan($vmid, 1); };
6789     warn $@ if $@;
6790 };
6791
6792 sub foreach_storage_used_by_vm {
6793     my ($conf, $func) = @_;
6794
6795     my $sidhash = {};
6796
6797     PVE::QemuConfig->foreach_volume($conf, sub {
6798         my ($ds, $drive) = @_;
6799         return if drive_is_cdrom($drive);
6800
6801         my $volid = $drive->{file};
6802
6803         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6804         $sidhash->{$sid} = $sid if $sid;
6805     });
6806
6807     foreach my $sid (sort keys %$sidhash) {
6808         &$func($sid);
6809     }
6810 }
6811
6812 my $qemu_snap_storage = {
6813     rbd => 1,
6814 };
6815 sub do_snapshots_with_qemu {
6816     my ($storecfg, $volid) = @_;
6817
6818     my $storage_name = PVE::Storage::parse_volume_id($volid);
6819     my $scfg = $storecfg->{ids}->{$storage_name};
6820     die "could not find storage '$storage_name'\n" if !defined($scfg);
6821
6822     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6823         return 1;
6824     }
6825
6826     if ($volid =~ m/\.(qcow2|qed)$/){
6827         return 1;
6828     }
6829
6830     return;
6831 }
6832
6833 sub qga_check_running {
6834     my ($vmid, $nowarn) = @_;
6835
6836     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6837     if ($@) {
6838         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6839         return 0;
6840     }
6841     return 1;
6842 }
6843
6844 sub template_create {
6845     my ($vmid, $conf, $disk) = @_;
6846
6847     my $storecfg = PVE::Storage::config();
6848
6849     PVE::QemuConfig->foreach_volume($conf, sub {
6850         my ($ds, $drive) = @_;
6851
6852         return if drive_is_cdrom($drive);
6853         return if $disk && $ds ne $disk;
6854
6855         my $volid = $drive->{file};
6856         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6857
6858         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6859         $drive->{file} = $voliddst;
6860         $conf->{$ds} = print_drive($drive);
6861         PVE::QemuConfig->write_config($vmid, $conf);
6862     });
6863 }
6864
6865 sub convert_iscsi_path {
6866     my ($path) = @_;
6867
6868     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6869         my $portal = $1;
6870         my $target = $2;
6871         my $lun = $3;
6872
6873         my $initiator_name = get_initiator_name();
6874
6875         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6876                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6877     }
6878
6879     die "cannot convert iscsi path '$path', unkown format\n";
6880 }
6881
6882 sub qemu_img_convert {
6883     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6884
6885     my $storecfg = PVE::Storage::config();
6886     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6887     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6888
6889     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6890
6891     my $cachemode;
6892     my $src_path;
6893     my $src_is_iscsi = 0;
6894     my $src_format;
6895
6896     if ($src_storeid) {
6897         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6898         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6899         $src_format = qemu_img_format($src_scfg, $src_volname);
6900         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6901         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6902         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6903     } elsif (-f $src_volid) {
6904         $src_path = $src_volid;
6905         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6906             $src_format = $1;
6907         }
6908     }
6909
6910     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6911
6912     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6913     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6914     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6915     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6916
6917     my $cmd = [];
6918     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6919     push @$cmd, '-l', "snapshot.name=$snapname"
6920         if $snapname && $src_format && $src_format eq "qcow2";
6921     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6922     push @$cmd, '-T', $cachemode if defined($cachemode);
6923
6924     if ($src_is_iscsi) {
6925         push @$cmd, '--image-opts';
6926         $src_path = convert_iscsi_path($src_path);
6927     } elsif ($src_format) {
6928         push @$cmd, '-f', $src_format;
6929     }
6930
6931     if ($dst_is_iscsi) {
6932         push @$cmd, '--target-image-opts';
6933         $dst_path = convert_iscsi_path($dst_path);
6934     } else {
6935         push @$cmd, '-O', $dst_format;
6936     }
6937
6938     push @$cmd, $src_path;
6939
6940     if (!$dst_is_iscsi && $is_zero_initialized) {
6941         push @$cmd, "zeroinit:$dst_path";
6942     } else {
6943         push @$cmd, $dst_path;
6944     }
6945
6946     my $parser = sub {
6947         my $line = shift;
6948         if($line =~ m/\((\S+)\/100\%\)/){
6949             my $percent = $1;
6950             my $transferred = int($size * $percent / 100);
6951             my $total_h = render_bytes($size, 1);
6952             my $transferred_h = render_bytes($transferred, 1);
6953
6954             print "transferred $transferred_h of $total_h ($percent%)";
6955         }
6956
6957     };
6958
6959     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6960     my $err = $@;
6961     die "copy failed: $err" if $err;
6962 }
6963
6964 sub qemu_img_format {
6965     my ($scfg, $volname) = @_;
6966
6967     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6968         return $1;
6969     } else {
6970         return "raw";
6971     }
6972 }
6973
6974 sub qemu_drive_mirror {
6975     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6976
6977     $jobs = {} if !$jobs;
6978
6979     my $qemu_target;
6980     my $format;
6981     $jobs->{"drive-$drive"} = {};
6982
6983     if ($dst_volid =~ /^nbd:/) {
6984         $qemu_target = $dst_volid;
6985         $format = "nbd";
6986     } else {
6987         my $storecfg = PVE::Storage::config();
6988         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6989
6990         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6991
6992         $format = qemu_img_format($dst_scfg, $dst_volname);
6993
6994         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6995
6996         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6997     }
6998
6999     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7000     $opts->{format} = $format if $format;
7001
7002     if (defined($src_bitmap)) {
7003         $opts->{sync} = 'incremental';
7004         $opts->{bitmap} = $src_bitmap;
7005         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7006     }
7007
7008     if (defined($bwlimit)) {
7009         $opts->{speed} = $bwlimit * 1024;
7010         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7011     } else {
7012         print "drive mirror is starting for drive-$drive\n";
7013     }
7014
7015     # if a job already runs for this device we get an error, catch it for cleanup
7016     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7017     if (my $err = $@) {
7018         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7019         warn "$@\n" if $@;
7020         die "mirroring error: $err\n";
7021     }
7022
7023     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7024 }
7025
7026 # $completion can be either
7027 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7028 # 'cancel': wait until all jobs are ready, block-job-cancel them
7029 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7030 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7031 sub qemu_drive_mirror_monitor {
7032     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7033
7034     $completion //= 'complete';
7035     $op //= "mirror";
7036
7037     eval {
7038         my $err_complete = 0;
7039
7040         my $starttime = time ();
7041         while (1) {
7042             die "block job ('$op') timed out\n" if $err_complete > 300;
7043
7044             my $stats = mon_cmd($vmid, "query-block-jobs");
7045             my $ctime = time();
7046
7047             my $running_jobs = {};
7048             for my $stat (@$stats) {
7049                 next if $stat->{type} ne $op;
7050                 $running_jobs->{$stat->{device}} = $stat;
7051             }
7052
7053             my $readycounter = 0;
7054
7055             for my $job_id (sort keys %$jobs) {
7056                 my $job = $running_jobs->{$job_id};
7057
7058                 my $vanished = !defined($job);
7059                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7060                 if($complete || ($vanished && $completion eq 'auto')) {
7061                     print "$job_id: $op-job finished\n";
7062                     delete $jobs->{$job_id};
7063                     next;
7064                 }
7065
7066                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7067
7068                 my $busy = $job->{busy};
7069                 my $ready = $job->{ready};
7070                 if (my $total = $job->{len}) {
7071                     my $transferred = $job->{offset} || 0;
7072                     my $remaining = $total - $transferred;
7073                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7074
7075                     my $duration = $ctime - $starttime;
7076                     my $total_h = render_bytes($total, 1);
7077                     my $transferred_h = render_bytes($transferred, 1);
7078
7079                     my $status = sprintf(
7080                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7081                         render_duration($duration),
7082                     );
7083
7084                     if ($ready) {
7085                         if ($busy) {
7086                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7087                         } else {
7088                             $status .= ", ready";
7089                         }
7090                     }
7091                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7092                     $jobs->{$job_id}->{ready} = $ready;
7093                 }
7094
7095                 $readycounter++ if $job->{ready};
7096             }
7097
7098             last if scalar(keys %$jobs) == 0;
7099
7100             if ($readycounter == scalar(keys %$jobs)) {
7101                 print "all '$op' jobs are ready\n";
7102
7103                 # do the complete later (or has already been done)
7104                 last if $completion eq 'skip' || $completion eq 'auto';
7105
7106                 if ($vmiddst && $vmiddst != $vmid) {
7107                     my $agent_running = $qga && qga_check_running($vmid);
7108                     if ($agent_running) {
7109                         print "freeze filesystem\n";
7110                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7111                     } else {
7112                         print "suspend vm\n";
7113                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7114                     }
7115
7116                     # if we clone a disk for a new target vm, we don't switch the disk
7117                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7118
7119                     if ($agent_running) {
7120                         print "unfreeze filesystem\n";
7121                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7122                     } else {
7123                         print "resume vm\n";
7124                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7125                     }
7126
7127                     last;
7128                 } else {
7129
7130                     for my $job_id (sort keys %$jobs) {
7131                         # try to switch the disk if source and destination are on the same guest
7132                         print "$job_id: Completing block job_id...\n";
7133
7134                         my $op;
7135                         if ($completion eq 'complete') {
7136                             $op = 'block-job-complete';
7137                         } elsif ($completion eq 'cancel') {
7138                             $op = 'block-job-cancel';
7139                         } else {
7140                             die "invalid completion value: $completion\n";
7141                         }
7142                         eval { mon_cmd($vmid, $op, device => $job_id) };
7143                         if ($@ =~ m/cannot be completed/) {
7144                             print "$job_id: block job cannot be completed, trying again.\n";
7145                             $err_complete++;
7146                         }else {
7147                             print "$job_id: Completed successfully.\n";
7148                             $jobs->{$job_id}->{complete} = 1;
7149                         }
7150                     }
7151                 }
7152             }
7153             sleep 1;
7154         }
7155     };
7156     my $err = $@;
7157
7158     if ($err) {
7159         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7160         die "block job ($op) error: $err";
7161     }
7162 }
7163
7164 sub qemu_blockjobs_cancel {
7165     my ($vmid, $jobs) = @_;
7166
7167     foreach my $job (keys %$jobs) {
7168         print "$job: Cancelling block job\n";
7169         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7170         $jobs->{$job}->{cancel} = 1;
7171     }
7172
7173     while (1) {
7174         my $stats = mon_cmd($vmid, "query-block-jobs");
7175
7176         my $running_jobs = {};
7177         foreach my $stat (@$stats) {
7178             $running_jobs->{$stat->{device}} = $stat;
7179         }
7180
7181         foreach my $job (keys %$jobs) {
7182
7183             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7184                 print "$job: Done.\n";
7185                 delete $jobs->{$job};
7186             }
7187         }
7188
7189         last if scalar(keys %$jobs) == 0;
7190
7191         sleep 1;
7192     }
7193 }
7194
7195 sub clone_disk {
7196     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7197         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7198
7199     my $newvolid;
7200
7201     if (!$full) {
7202         print "create linked clone of drive $drivename ($drive->{file})\n";
7203         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7204         push @$newvollist, $newvolid;
7205     } else {
7206
7207         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7208         $storeid = $storage if $storage;
7209
7210         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7211
7212         print "create full clone of drive $drivename ($drive->{file})\n";
7213         my $name = undef;
7214         my $size = undef;
7215         if (drive_is_cloudinit($drive)) {
7216             $name = "vm-$newvmid-cloudinit";
7217             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7218             if ($scfg->{path}) {
7219                 $name .= ".$dst_format";
7220             }
7221             $snapname = undef;
7222             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7223         } elsif ($drivename eq 'efidisk0') {
7224             $size = get_efivars_size($conf);
7225         } else {
7226             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7227         }
7228         $newvolid = PVE::Storage::vdisk_alloc(
7229             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7230         );
7231         push @$newvollist, $newvolid;
7232
7233         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7234
7235         if (drive_is_cloudinit($drive)) {
7236             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7237             # if this is the case, we have to complete any block-jobs still there from
7238             # previous drive-mirrors
7239             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7240                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7241             }
7242             goto no_data_clone;
7243         }
7244
7245         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7246         if (!$running || $snapname) {
7247             # TODO: handle bwlimits
7248             if ($drivename eq 'efidisk0') {
7249                 # the relevant data on the efidisk may be smaller than the source
7250                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7251                 # that is given by the OVMF_VARS.fd
7252                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7253                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7254
7255                 # better for Ceph if block size is not too small, see bug #3324
7256                 my $bs = 1024*1024;
7257
7258                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7259                     "if=$src_path", "of=$dst_path"]);
7260             } else {
7261                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7262             }
7263         } else {
7264
7265             my $kvmver = get_running_qemu_version ($vmid);
7266             if (!min_version($kvmver, 2, 7)) {
7267                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7268                     if $drive->{iothread};
7269             }
7270
7271             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7272                 $completion, $qga, $bwlimit);
7273         }
7274     }
7275
7276 no_data_clone:
7277     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7278
7279     my $disk = $drive;
7280     $disk->{format} = undef;
7281     $disk->{file} = $newvolid;
7282     $disk->{size} = $size if defined($size);
7283
7284     return $disk;
7285 }
7286
7287 sub get_running_qemu_version {
7288     my ($vmid) = @_;
7289     my $res = mon_cmd($vmid, "query-version");
7290     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7291 }
7292
7293 sub qemu_use_old_bios_files {
7294     my ($machine_type) = @_;
7295
7296     return if !$machine_type;
7297
7298     my $use_old_bios_files = undef;
7299
7300     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7301         $machine_type = $1;
7302         $use_old_bios_files = 1;
7303     } else {
7304         my $version = extract_version($machine_type, kvm_user_version());
7305         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7306         # load new efi bios files on migration. So this hack is required to allow
7307         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7308         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7309         $use_old_bios_files = !min_version($version, 2, 4);
7310     }
7311
7312     return ($use_old_bios_files, $machine_type);
7313 }
7314
7315 sub get_efivars_size {
7316     my ($conf) = @_;
7317     my $arch = get_vm_arch($conf);
7318     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7319     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7320     return -s $ovmf_vars;
7321 }
7322
7323 sub update_efidisk_size {
7324     my ($conf) = @_;
7325
7326     return if !defined($conf->{efidisk0});
7327
7328     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7329     $disk->{size} = get_efivars_size($conf);
7330     $conf->{efidisk0} = print_drive($disk);
7331
7332     return;
7333 }
7334
7335 sub create_efidisk($$$$$) {
7336     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7337
7338     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7339     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7340
7341     my $vars_size_b = -s $ovmf_vars;
7342     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7343     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7344     PVE::Storage::activate_volumes($storecfg, [$volid]);
7345
7346     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7347     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7348
7349     return ($volid, $size/1024);
7350 }
7351
7352 sub vm_iothreads_list {
7353     my ($vmid) = @_;
7354
7355     my $res = mon_cmd($vmid, 'query-iothreads');
7356
7357     my $iothreads = {};
7358     foreach my $iothread (@$res) {
7359         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7360     }
7361
7362     return $iothreads;
7363 }
7364
7365 sub scsihw_infos {
7366     my ($conf, $drive) = @_;
7367
7368     my $maxdev = 0;
7369
7370     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7371         $maxdev = 7;
7372     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7373         $maxdev = 1;
7374     } else {
7375         $maxdev = 256;
7376     }
7377
7378     my $controller = int($drive->{index} / $maxdev);
7379     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7380         ? "virtioscsi"
7381         : "scsihw";
7382
7383     return ($maxdev, $controller, $controller_prefix);
7384 }
7385
7386 sub windows_version {
7387     my ($ostype) = @_;
7388
7389     return 0 if !$ostype;
7390
7391     my $winversion = 0;
7392
7393     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7394         $winversion = 5;
7395     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7396         $winversion = 6;
7397     } elsif ($ostype =~ m/^win(\d+)$/) {
7398         $winversion = $1;
7399     }
7400
7401     return $winversion;
7402 }
7403
7404 sub resolve_dst_disk_format {
7405         my ($storecfg, $storeid, $src_volname, $format) = @_;
7406         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7407
7408         if (!$format) {
7409             # if no target format is specified, use the source disk format as hint
7410             if ($src_volname) {
7411                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7412                 $format = qemu_img_format($scfg, $src_volname);
7413             } else {
7414                 return $defFormat;
7415             }
7416         }
7417
7418         # test if requested format is supported - else use default
7419         my $supported = grep { $_ eq $format } @$validFormats;
7420         $format = $defFormat if !$supported;
7421         return $format;
7422 }
7423
7424 # NOTE: if this logic changes, please update docs & possibly gui logic
7425 sub find_vmstate_storage {
7426     my ($conf, $storecfg) = @_;
7427
7428     # first, return storage from conf if set
7429     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7430
7431     my ($target, $shared, $local);
7432
7433     foreach_storage_used_by_vm($conf, sub {
7434         my ($sid) = @_;
7435         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7436         my $dst = $scfg->{shared} ? \$shared : \$local;
7437         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7438     });
7439
7440     # second, use shared storage where VM has at least one disk
7441     # third, use local storage where VM has at least one disk
7442     # fall back to local storage
7443     $target = $shared // $local // 'local';
7444
7445     return $target;
7446 }
7447
7448 sub generate_uuid {
7449     my ($uuid, $uuid_str);
7450     UUID::generate($uuid);
7451     UUID::unparse($uuid, $uuid_str);
7452     return $uuid_str;
7453 }
7454
7455 sub generate_smbios1_uuid {
7456     return "uuid=".generate_uuid();
7457 }
7458
7459 sub nbd_stop {
7460     my ($vmid) = @_;
7461
7462     mon_cmd($vmid, 'nbd-server-stop');
7463 }
7464
7465 sub create_reboot_request {
7466     my ($vmid) = @_;
7467     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7468         or die "failed to create reboot trigger file: $!\n";
7469     close($fh);
7470 }
7471
7472 sub clear_reboot_request {
7473     my ($vmid) = @_;
7474     my $path = "/run/qemu-server/$vmid.reboot";
7475     my $res = 0;
7476
7477     $res = unlink($path);
7478     die "could not remove reboot request for $vmid: $!"
7479         if !$res && $! != POSIX::ENOENT;
7480
7481     return $res;
7482 }
7483
7484 sub bootorder_from_legacy {
7485     my ($conf, $bootcfg) = @_;
7486
7487     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7488     my $bootindex_hash = {};
7489     my $i = 1;
7490     foreach my $o (split(//, $boot)) {
7491         $bootindex_hash->{$o} = $i*100;
7492         $i++;
7493     }
7494
7495     my $bootorder = {};
7496
7497     PVE::QemuConfig->foreach_volume($conf, sub {
7498         my ($ds, $drive) = @_;
7499
7500         if (drive_is_cdrom ($drive, 1)) {
7501             if ($bootindex_hash->{d}) {
7502                 $bootorder->{$ds} = $bootindex_hash->{d};
7503                 $bootindex_hash->{d} += 1;
7504             }
7505         } elsif ($bootindex_hash->{c}) {
7506             $bootorder->{$ds} = $bootindex_hash->{c}
7507                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7508             $bootindex_hash->{c} += 1;
7509         }
7510     });
7511
7512     if ($bootindex_hash->{n}) {
7513         for (my $i = 0; $i < $MAX_NETS; $i++) {
7514             my $netname = "net$i";
7515             next if !$conf->{$netname};
7516             $bootorder->{$netname} = $bootindex_hash->{n};
7517             $bootindex_hash->{n} += 1;
7518         }
7519     }
7520
7521     return $bootorder;
7522 }
7523
7524 # Generate default device list for 'boot: order=' property. Matches legacy
7525 # default boot order, but with explicit device names. This is important, since
7526 # the fallback for when neither 'order' nor the old format is specified relies
7527 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7528 sub get_default_bootdevices {
7529     my ($conf) = @_;
7530
7531     my @ret = ();
7532
7533     # harddisk
7534     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7535     push @ret, $first if $first;
7536
7537     # cdrom
7538     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7539     push @ret, $first if $first;
7540
7541     # network
7542     for (my $i = 0; $i < $MAX_NETS; $i++) {
7543         my $netname = "net$i";
7544         next if !$conf->{$netname};
7545         push @ret, $netname;
7546         last;
7547     }
7548
7549     return \@ret;
7550 }
7551
7552 sub device_bootorder {
7553     my ($conf) = @_;
7554
7555     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7556
7557     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7558
7559     my $bootorder = {};
7560     if (!defined($boot) || $boot->{legacy}) {
7561         $bootorder = bootorder_from_legacy($conf, $boot);
7562     } elsif ($boot->{order}) {
7563         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7564         for my $dev (PVE::Tools::split_list($boot->{order})) {
7565             $bootorder->{$dev} = $i++;
7566         }
7567     }
7568
7569     return $bootorder;
7570 }
7571
7572 sub register_qmeventd_handle {
7573     my ($vmid) = @_;
7574
7575     my $fh;
7576     my $peer = "/var/run/qmeventd.sock";
7577     my $count = 0;
7578
7579     for (;;) {
7580         $count++;
7581         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7582         last if $fh;
7583         if ($! != EINTR && $! != EAGAIN) {
7584             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7585         }
7586         if ($count > 4) {
7587             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7588               . "after $count retries\n";
7589         }
7590         usleep(25000);
7591     }
7592
7593     # send handshake to mark VM as backing up
7594     print $fh to_json({vzdump => {vmid => "$vmid"}});
7595
7596     # return handle to be closed later when inhibit is no longer required
7597     return $fh;
7598 }
7599
7600 # bash completion helper
7601
7602 sub complete_backup_archives {
7603     my ($cmdname, $pname, $cvalue) = @_;
7604
7605     my $cfg = PVE::Storage::config();
7606
7607     my $storeid;
7608
7609     if ($cvalue =~ m/^([^:]+):/) {
7610         $storeid = $1;
7611     }
7612
7613     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7614
7615     my $res = [];
7616     foreach my $id (keys %$data) {
7617         foreach my $item (@{$data->{$id}}) {
7618             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7619             push @$res, $item->{volid} if defined($item->{volid});
7620         }
7621     }
7622
7623     return $res;
7624 }
7625
7626 my $complete_vmid_full = sub {
7627     my ($running) = @_;
7628
7629     my $idlist = vmstatus();
7630
7631     my $res = [];
7632
7633     foreach my $id (keys %$idlist) {
7634         my $d = $idlist->{$id};
7635         if (defined($running)) {
7636             next if $d->{template};
7637             next if $running && $d->{status} ne 'running';
7638             next if !$running && $d->{status} eq 'running';
7639         }
7640         push @$res, $id;
7641
7642     }
7643     return $res;
7644 };
7645
7646 sub complete_vmid {
7647     return &$complete_vmid_full();
7648 }
7649
7650 sub complete_vmid_stopped {
7651     return &$complete_vmid_full(0);
7652 }
7653
7654 sub complete_vmid_running {
7655     return &$complete_vmid_full(1);
7656 }
7657
7658 sub complete_storage {
7659
7660     my $cfg = PVE::Storage::config();
7661     my $ids = $cfg->{ids};
7662
7663     my $res = [];
7664     foreach my $sid (keys %$ids) {
7665         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7666         next if !$ids->{$sid}->{content}->{images};
7667         push @$res, $sid;
7668     }
7669
7670     return $res;
7671 }
7672
7673 sub complete_migration_storage {
7674     my ($cmd, $param, $current_value, $all_args) = @_;
7675
7676     my $targetnode = @$all_args[1];
7677
7678     my $cfg = PVE::Storage::config();
7679     my $ids = $cfg->{ids};
7680
7681     my $res = [];
7682     foreach my $sid (keys %$ids) {
7683         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7684         next if !$ids->{$sid}->{content}->{images};
7685         push @$res, $sid;
7686     }
7687
7688     return $res;
7689 }
7690
7691 sub vm_is_paused {
7692     my ($vmid) = @_;
7693     my $qmpstatus = eval {
7694         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7695         mon_cmd($vmid, "query-status");
7696     };
7697     warn "$@\n" if $@;
7698     return $qmpstatus && $qmpstatus->{status} eq "paused";
7699 }
7700
7701 1;