PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 369     },
 370     ostype => {
 371         optional => 1,
 372         type => 'string',
 373         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 374         description => "Specify guest operating system.",
 375         verbose_description => <<EODESC,
 376 Specify guest operating system. This is used to enable special
 377 optimization/features for specific operating systems:
 378
 379 [horizontal]
 380 other;; unspecified OS
 381 wxp;; Microsoft Windows XP
 382 w2k;; Microsoft Windows 2000
 383 w2k3;; Microsoft Windows 2003
 384 w2k8;; Microsoft Windows 2008
 385 wvista;; Microsoft Windows Vista
 386 win7;; Microsoft Windows 7
 387 win8;; Microsoft Windows 8/2012/2012r2
 388 win10;; Microsoft Windows 10/2016/2019
 389 l24;; Linux 2.4 Kernel
 390 l26;; Linux 2.6 - 5.X Kernel
 391 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 392 EODESC
 393     },
 394     boot => {
 395         optional => 1,
 396         type => 'string', format => 'pve-qm-boot',
 397         description => "Specify guest boot order. Use with 'order=', usage with"
 398                      . " no key or 'legacy=' is deprecated.",
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     keephugepages => {
 440         optional => 1,
 441         type => 'boolean',
 442         default => 0,
 443         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 444             ." after VM shutdown and can be used for subsequent starts.",
 445     },
 446     vcpus => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "Number of hotplugged vcpus.",
 450         minimum => 1,
 451         default => 0,
 452     },
 453     acpi => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Enable/disable ACPI.",
 457         default => 1,
 458     },
 459     agent => {
 460         optional => 1,
 461         description => "Enable/disable Qemu GuestAgent and its properties.",
 462         type => 'string',
 463         format => $agent_fmt,
 464     },
 465     kvm => {
 466         optional => 1,
 467         type => 'boolean',
 468         description => "Enable/disable KVM hardware virtualization.",
 469         default => 1,
 470     },
 471     tdf => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable time drift fix.",
 475         default => 0,
 476     },
 477     localtime => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Set the real time clock to local time. This is enabled by default if ostype"
 481             ." indicates a Microsoft OS.",
 482     },
 483     freeze => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 487     },
 488     vga => {
 489         optional => 1,
 490         type => 'string', format => $vga_fmt,
 491         description => "Configure the VGA hardware.",
 492         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 493             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 494             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 495             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 496             ." display server. For win* OS you can select how many independent displays you want,"
 497             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 498             ." using a serial device as terminal.",
 499     },
 500     watchdog => {
 501         optional => 1,
 502         type => 'string', format => 'pve-qm-watchdog',
 503         description => "Create a virtual hardware watchdog device.",
 504         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 505             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 506             ." else the watchdog will reset the guest (or execute the respective action specified)",
 507     },
 508     startdate => {
 509         optional => 1,
 510         type => 'string',
 511         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 512         description => "Set the initial date of the real time clock. Valid format for date are:"
 513             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 514         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 515         default => 'now',
 516     },
 517     startup =>  get_standard_option('pve-startup-order'),
 518     template => {
 519         optional => 1,
 520         type => 'boolean',
 521         description => "Enable/disable Template.",
 522         default => 0,
 523     },
 524     args => {
 525         optional => 1,
 526         type => 'string',
 527         description => "Arbitrary arguments passed to kvm.",
 528         verbose_description => <<EODESCR,
 529 Arbitrary arguments passed to kvm, for example:
 530
 531 args: -no-reboot -no-hpet
 532
 533 NOTE: this option is for experts only.
 534 EODESCR
 535     },
 536     tablet => {
 537         optional => 1,
 538         type => 'boolean',
 539         default => 1,
 540         description => "Enable/disable the USB tablet device.",
 541         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 542             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 543             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 544             ." may consider disabling this to save some context switches. This is turned off by"
 545             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 546     },
 547     migrate_speed => {
 548         optional => 1,
 549         type => 'integer',
 550         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 551         minimum => 0,
 552         default => 0,
 553     },
 554     migrate_downtime => {
 555         optional => 1,
 556         type => 'number',
 557         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 558         minimum => 0,
 559         default => 0.1,
 560     },
 561     cdrom => {
 562         optional => 1,
 563         type => 'string', format => 'pve-qm-ide',
 564         typetext => '<volume>',
 565         description => "This is an alias for option -ide2",
 566     },
 567     cpu => {
 568         optional => 1,
 569         description => "Emulated CPU type.",
 570         type => 'string',
 571         format => 'pve-vm-cpu-conf',
 572     },
 573     parent => get_standard_option('pve-snapshot-name', {
 574         optional => 1,
 575         description => "Parent snapshot name. This is used internally, and should not be modified.",
 576     }),
 577     snaptime => {
 578         optional => 1,
 579         description => "Timestamp for snapshots.",
 580         type => 'integer',
 581         minimum => 0,
 582     },
 583     vmstate => {
 584         optional => 1,
 585         type => 'string', format => 'pve-volume-id',
 586         description => "Reference to a volume which stores the VM state. This is used internally"
 587             ." for snapshots.",
 588     },
 589     vmstatestorage => get_standard_option('pve-storage-id', {
 590         description => "Default storage for VM state volumes/files.",
 591         optional => 1,
 592     }),
 593     runningmachine => get_standard_option('pve-qemu-machine', {
 594         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 595             ." for snapshots.",
 596     }),
 597     runningcpu => {
 598         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 599             ." internally for snapshots.",
 600         optional => 1,
 601         type => 'string',
 602         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 603         format_description => 'QEMU -cpu parameter'
 604     },
 605     machine => get_standard_option('pve-qemu-machine'),
 606     arch => {
 607         description => "Virtual processor architecture. Defaults to the host.",
 608         optional => 1,
 609         type => 'string',
 610         enum => [qw(x86_64 aarch64)],
 611     },
 612     smbios1 => {
 613         description => "Specify SMBIOS type 1 fields.",
 614         type => 'string', format => 'pve-qm-smbios1',
 615         maxLength => 512,
 616         optional => 1,
 617     },
 618     protection => {
 619         optional => 1,
 620         type => 'boolean',
 621         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 622             ." remove disk operations.",
 623         default => 0,
 624     },
 625     bios => {
 626         optional => 1,
 627         type => 'string',
 628         enum => [ qw(seabios ovmf) ],
 629         description => "Select BIOS implementation.",
 630         default => 'seabios',
 631     },
 632     vmgenid => {
 633         type => 'string',
 634         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 635         format_description => 'UUID',
 636         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 637             ." to disable explicitly.",
 638         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 639             ." value identifier to the guest OS. This allows to notify the guest operating system"
 640             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 641             ." execution or creation from a template). The guest operating system notices the"
 642             ." change, and is then able to react as appropriate by marking its copies of"
 643             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 644             ."Note that auto-creation only works when done through API/CLI create or update methods"
 645             .", but not when manually editing the config file.",
 646         default => "1 (autogenerated)",
 647         optional => 1,
 648     },
 649     hookscript => {
 650         type => 'string',
 651         format => 'pve-volume-id',
 652         optional => 1,
 653         description => "Script that will be executed during various steps in the vms lifetime.",
 654     },
 655     ivshmem => {
 656         type => 'string',
 657         format => $ivshmem_fmt,
 658         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 659             ." the host.",
 660         optional => 1,
 661     },
 662     audio0 => {
 663         type => 'string',
 664         format => $audio_fmt,
 665         description => "Configure a audio device, useful in combination with QXL/Spice.",
 666         optional => 1
 667     },
 668     spice_enhancements => {
 669         type => 'string',
 670         format => $spice_enhancements_fmt,
 671         description => "Configure additional enhancements for SPICE.",
 672         optional => 1
 673     },
 674     tags => {
 675         type => 'string', format => 'pve-tag-list',
 676         description => 'Tags of the VM. This is only meta information.',
 677         optional => 1,
 678     },
 679     rng0 => {
 680         type => 'string',
 681         format => $rng_fmt,
 682         description => "Configure a VirtIO-based Random Number Generator.",
 683         optional => 1,
 684     },
 685 };
 686
 687 my $cicustom_fmt = {
 688     meta => {
 689         type => 'string',
 690         optional => 1,
 691         description => 'Specify a custom file containing all meta data passed to the VM via"
 692             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 693         format => 'pve-volume-id',
 694         format_description => 'volume',
 695     },
 696     network => {
 697         type => 'string',
 698         optional => 1,
 699         description => 'Specify a custom file containing all network data passed to the VM via'
 700             .' cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704     user => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all user data passed to the VM via'
 708             .' cloud-init.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712 };
 713 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 714
 715 my $confdesc_cloudinit = {
 716     citype => {
 717         optional => 1,
 718         type => 'string',
 719         description => 'Specifies the cloud-init configuration format. The default depends on the'
 720             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 721             .' and `configdrive2` for windows.',
 722         enum => ['configdrive2', 'nocloud', 'opennebula'],
 723     },
 724     ciuser => {
 725         optional => 1,
 726         type => 'string',
 727         description => "cloud-init: User name to change ssh keys and password for instead of the"
 728             ." image's configured default user.",
 729     },
 730     cipassword => {
 731         optional => 1,
 732         type => 'string',
 733         description => 'cloud-init: Password to assign the user. Using this is generally not'
 734             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 735             .' support hashed passwords.',
 736     },
 737     cicustom => {
 738         optional => 1,
 739         type => 'string',
 740         description => 'cloud-init: Specify custom files to replace the automatically generated'
 741             .' ones at start.',
 742         format => 'pve-qm-cicustom',
 743     },
 744     searchdomain => {
 745         optional => 1,
 746         type => 'string',
 747         description => "cloud-init: Sets DNS search domains for a container. Create will'
 748             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 749             .' are set.",
 750     },
 751     nameserver => {
 752         optional => 1,
 753         type => 'string', format => 'address-list',
 754         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 755             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 756             .' are set.",
 757     },
 758     sshkeys => {
 759         optional => 1,
 760         type => 'string',
 761         format => 'urlencoded',
 762         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 763     },
 764 };
 765
 766 # what about other qemu settings ?
 767 #cpu => 'string',
 768 #machine => 'string',
 769 #fda => 'file',
 770 #fdb => 'file',
 771 #mtdblock => 'file',
 772 #sd => 'file',
 773 #pflash => 'file',
 774 #snapshot => 'bool',
 775 #bootp => 'file',
 776 ##tftp => 'dir',
 777 ##smb => 'dir',
 778 #kernel => 'file',
 779 #append => 'string',
 780 #initrd => 'file',
 781 ##soundhw => 'string',
 782
 783 while (my ($k, $v) = each %$confdesc) {
 784     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 785 }
 786
 787 my $MAX_USB_DEVICES = 5;
 788 my $MAX_NETS = 32;
 789 my $MAX_SERIAL_PORTS = 4;
 790 my $MAX_PARALLEL_PORTS = 3;
 791 my $MAX_NUMA = 8;
 792
 793 my $numa_fmt = {
 794     cpus => {
 795         type => "string",
 796         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 797         description => "CPUs accessing this NUMA node.",
 798         format_description => "id[-id];...",
 799     },
 800     memory => {
 801         type => "number",
 802         description => "Amount of memory this NUMA node provides.",
 803         optional => 1,
 804     },
 805     hostnodes => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "Host NUMA nodes to use.",
 809         format_description => "id[-id];...",
 810         optional => 1,
 811     },
 812     policy => {
 813         type => 'string',
 814         enum => [qw(preferred bind interleave)],
 815         description => "NUMA allocation policy.",
 816         optional => 1,
 817     },
 818 };
 819 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 820 my $numadesc = {
 821     optional => 1,
 822     type => 'string', format => $numa_fmt,
 823     description => "NUMA topology.",
 824 };
 825 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 826
 827 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 828     $confdesc->{"numa$i"} = $numadesc;
 829 }
 830
 831 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 832                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 833                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 834 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 835
 836 my $net_fmt_bridge_descr = <<__EOD__;
 837 Bridge to attach the network device to. The Proxmox VE standard bridge
 838 is called 'vmbr0'.
 839
 840 If you do not specify a bridge, we create a kvm user (NATed) network
 841 device, which provides DHCP and DNS services. The following addresses
 842 are used:
 843
 844  10.0.2.2   Gateway
 845  10.0.2.3   DNS Server
 846  10.0.2.4   SMB Server
 847
 848 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 849 __EOD__
 850
 851 my $net_fmt = {
 852     macaddr  => get_standard_option('mac-addr', {
 853         description => "MAC address. That address must be unique withing your network. This is"
 854             ." automatically generated if not specified.",
 855     }),
 856     model => {
 857         type => 'string',
 858         description => "Network Card Model. The 'virtio' model provides the best performance with"
 859             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 860             ." best to use 'e1000'.",
 861         enum => $nic_model_list,
 862         default_key => 1,
 863     },
 864     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 865     bridge => {
 866         type => 'string',
 867         description => $net_fmt_bridge_descr,
 868         format_description => 'bridge',
 869         pattern => '[-_.\w\d]+',
 870         optional => 1,
 871     },
 872     queues => {
 873         type => 'integer',
 874         minimum => 0, maximum => 16,
 875         description => 'Number of packet queues to be used on the device.',
 876         optional => 1,
 877     },
 878     rate => {
 879         type => 'number',
 880         minimum => 0,
 881         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 882         optional => 1,
 883     },
 884     tag => {
 885         type => 'integer',
 886         minimum => 1, maximum => 4094,
 887         description => 'VLAN tag to apply to packets on this interface.',
 888         optional => 1,
 889     },
 890     trunks => {
 891         type => 'string',
 892         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 893         description => 'VLAN trunks to pass through this interface.',
 894         format_description => 'vlanid[;vlanid...]',
 895         optional => 1,
 896     },
 897     firewall => {
 898         type => 'boolean',
 899         description => 'Whether this interface should be protected by the firewall.',
 900         optional => 1,
 901     },
 902     link_down => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be disconnected (like pulling the plug).',
 905         optional => 1,
 906     },
 907     mtu => {
 908         type => 'integer',
 909         minimum => 1, maximum => 65520,
 910         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 911         optional => 1,
 912     },
 913 };
 914
 915 my $netdesc = {
 916     optional => 1,
 917     type => 'string', format => $net_fmt,
 918     description => "Specify network devices.",
 919 };
 920
 921 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 922
 923 my $ipconfig_fmt = {
 924     ip => {
 925         type => 'string',
 926         format => 'pve-ipv4-config',
 927         format_description => 'IPv4Format/CIDR',
 928         description => 'IPv4 address in CIDR format.',
 929         optional => 1,
 930         default => 'dhcp',
 931     },
 932     gw => {
 933         type => 'string',
 934         format => 'ipv4',
 935         format_description => 'GatewayIPv4',
 936         description => 'Default gateway for IPv4 traffic.',
 937         optional => 1,
 938         requires => 'ip',
 939     },
 940     ip6 => {
 941         type => 'string',
 942         format => 'pve-ipv6-config',
 943         format_description => 'IPv6Format/CIDR',
 944         description => 'IPv6 address in CIDR format.',
 945         optional => 1,
 946         default => 'dhcp',
 947     },
 948     gw6 => {
 949         type => 'string',
 950         format => 'ipv6',
 951         format_description => 'GatewayIPv6',
 952         description => 'Default gateway for IPv6 traffic.',
 953         optional => 1,
 954         requires => 'ip6',
 955     },
 956 };
 957 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 958 my $ipconfigdesc = {
 959     optional => 1,
 960     type => 'string', format => 'pve-qm-ipconfig',
 961     description => <<'EODESCR',
 962 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 963
 964 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 965
 966 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 967 gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 969 cloud-init 19.4 or newer.
 970
 971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 972 dhcp on IPv4.
 973 EODESCR
 974 };
 975 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 976
 977 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 978     $confdesc->{"net$i"} = $netdesc;
 979     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 980 }
 981
 982 foreach my $key (keys %$confdesc_cloudinit) {
 983     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 984 }
 985
 986 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 987 sub verify_volume_id_or_qm_path {
 988     my ($volid, $noerr) = @_;
 989
 990     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 991         return $volid;
 992     }
 993
 994     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 995     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 996     if ($@) {
 997         return if $noerr;
 998         die $@;
 999     }
1000     return $volid;
1001 }
1002
1003 my $usb_fmt = {
1004     host => {
1005         default_key => 1,
1006         type => 'string', format => 'pve-qm-usb-device',
1007         format_description => 'HOSTUSBDEVICE|spice',
1008         description => <<EODESCR,
1009 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1010
1011  'bus-port(.port)*' (decimal numbers) or
1012  'vendor_id:product_id' (hexadeciaml numbers) or
1013  'spice'
1014
1015 You can use the 'lsusb -t' command to list existing usb devices.
1016
1017 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1018 machines - use with special care.
1019
1020 The value 'spice' can be used to add a usb redirection devices for spice.
1021 EODESCR
1022     },
1023     usb3 => {
1024         optional => 1,
1025         type => 'boolean',
1026         description => "Specifies whether if given host option is a USB3 device or port.",
1027         default => 0,
1028     },
1029 };
1030
1031 my $usbdesc = {
1032     optional => 1,
1033     type => 'string', format => $usb_fmt,
1034     description => "Configure an USB device (n is 0 to 4).",
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1037
1038 my $serialdesc = {
1039         optional => 1,
1040         type => 'string',
1041         pattern => '(/dev/.+|socket)',
1042         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1043         verbose_description =>  <<EODESCR,
1044 Create a serial device inside the VM (n is 0 to 3), and pass through a
1045 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046 host side (use 'qm terminal' to open a terminal connection).
1047
1048 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1049 use with special care.
1050
1051 CAUTION: Experimental! User reported problems with this option.
1052 EODESCR
1053 };
1054
1055 my $paralleldesc= {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1059         description =>  "Map host parallel devices (n is 0 to 2).",
1060         verbose_description =>  <<EODESCR,
1061 Map host parallel devices (n is 0 to 2).
1062
1063 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1064 machines - use with special care.
1065
1066 CAUTION: Experimental! User reported problems with this option.
1067 EODESCR
1068 };
1069
1070 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1071     $confdesc->{"parallel$i"} = $paralleldesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1075     $confdesc->{"serial$i"} = $serialdesc;
1076 }
1077
1078 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1079     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1080 }
1081
1082 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1083     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1084 }
1085
1086 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1087     $confdesc->{"usb$i"} = $usbdesc;
1088 }
1089
1090 my $boot_fmt = {
1091     legacy => {
1092         optional => 1,
1093         default_key => 1,
1094         type => 'string',
1095         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1096                      . " Deprecated, use 'order=' instead.",
1097         pattern => '[acdn]{1,4}',
1098         format_description => "[acdn]{1,4}",
1099
1100         # note: this is also the fallback if boot: is not given at all
1101         default => 'cdn',
1102     },
1103     order => {
1104         optional => 1,
1105         type => 'string',
1106         format => 'pve-qm-bootdev-list',
1107         format_description => "device[;device...]",
1108         description => <<EODESC,
1109 The guest will attempt to boot from devices in the order they appear here.
1110
1111 Disks, optical drives and passed-through storage USB devices will be directly
1112 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1113 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1114
1115 Note that only devices in this list will be marked as bootable and thus loaded
1116 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1117 (e.g. software-raid), you need to specify all of them here.
1118
1119 Overrides the deprecated 'legacy=[acdn]*' value when given.
1120 EODESC
1121     },
1122 };
1123 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1124
1125 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1126 sub verify_bootdev {
1127     my ($dev, $noerr) = @_;
1128
1129     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1130
1131     my $check = sub {
1132         my ($base) = @_;
1133         return 0 if $dev !~ m/^$base\d+$/;
1134         return 0 if !$confdesc->{$dev};
1135         return 1;
1136     };
1137
1138     return $dev if $check->("net");
1139     return $dev if $check->("usb");
1140     return $dev if $check->("hostpci");
1141
1142     return if $noerr;
1143     die "invalid boot device '$dev'\n";
1144 }
1145
1146 sub print_bootorder {
1147     my ($devs) = @_;
1148     return "" if !@$devs;
1149     my $data = { order => join(';', @$devs) };
1150     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1151 }
1152
1153 my $kvm_api_version = 0;
1154
1155 sub kvm_version {
1156     return $kvm_api_version if $kvm_api_version;
1157
1158     open my $fh, '<', '/dev/kvm' or return;
1159
1160     # 0xae00 => KVM_GET_API_VERSION
1161     $kvm_api_version = ioctl($fh, 0xae00, 0);
1162     close($fh);
1163
1164     return $kvm_api_version;
1165 }
1166
1167 my $kvm_user_version = {};
1168 my $kvm_mtime = {};
1169
1170 sub kvm_user_version {
1171     my ($binary) = @_;
1172
1173     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1174     my $st = stat($binary);
1175
1176     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1177     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1178         $cachedmtime == $st->mtime;
1179
1180     $kvm_user_version->{$binary} = 'unknown';
1181     $kvm_mtime->{$binary} = $st->mtime;
1182
1183     my $code = sub {
1184         my $line = shift;
1185         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186             $kvm_user_version->{$binary} = $2;
1187         }
1188     };
1189
1190     eval { run_command([$binary, '--version'], outfunc => $code); };
1191     warn $@ if $@;
1192
1193     return $kvm_user_version->{$binary};
1194
1195 }
1196 my sub extract_version {
1197     my ($machine_type, $version) = @_;
1198     $version = kvm_user_version() if !defined($version);
1199     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1200 }
1201
1202 sub kernel_has_vhost_net {
1203     return -c '/dev/vhost-net';
1204 }
1205
1206 sub option_exists {
1207     my $key = shift;
1208     return defined($confdesc->{$key});
1209 }
1210
1211 my $cdrom_path;
1212 sub get_cdrom_path {
1213
1214     return  $cdrom_path if $cdrom_path;
1215
1216     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1217     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1218     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1219 }
1220
1221 sub get_iso_path {
1222     my ($storecfg, $vmid, $cdrom) = @_;
1223
1224     if ($cdrom eq 'cdrom') {
1225         return get_cdrom_path();
1226     } elsif ($cdrom eq 'none') {
1227         return '';
1228     } elsif ($cdrom =~ m|^/|) {
1229         return $cdrom;
1230     } else {
1231         return PVE::Storage::path($storecfg, $cdrom);
1232     }
1233 }
1234
1235 # try to convert old style file names to volume IDs
1236 sub filename_to_volume_id {
1237     my ($vmid, $file, $media) = @_;
1238
1239      if (!($file eq 'none' || $file eq 'cdrom' ||
1240           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1241
1242         return if $file =~ m|/|;
1243
1244         if ($media && $media eq 'cdrom') {
1245             $file = "local:iso/$file";
1246         } else {
1247             $file = "local:$vmid/$file";
1248         }
1249     }
1250
1251     return $file;
1252 }
1253
1254 sub verify_media_type {
1255     my ($opt, $vtype, $media) = @_;
1256
1257     return if !$media;
1258
1259     my $etype;
1260     if ($media eq 'disk') {
1261         $etype = 'images';
1262     } elsif ($media eq 'cdrom') {
1263         $etype = 'iso';
1264     } else {
1265         die "internal error";
1266     }
1267
1268     return if ($vtype eq $etype);
1269
1270     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1271 }
1272
1273 sub cleanup_drive_path {
1274     my ($opt, $storecfg, $drive) = @_;
1275
1276     # try to convert filesystem paths to volume IDs
1277
1278     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1279         ($drive->{file} !~ m|^/dev/.+|) &&
1280         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1281         ($drive->{file} !~ m/^\d+$/)) {
1282         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1283         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1284             if !$vtype;
1285         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1286         verify_media_type($opt, $vtype, $drive->{media});
1287         $drive->{file} = $volid;
1288     }
1289
1290     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1291 }
1292
1293 sub parse_hotplug_features {
1294     my ($data) = @_;
1295
1296     my $res = {};
1297
1298     return $res if $data eq '0';
1299
1300     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1301
1302     foreach my $feature (PVE::Tools::split_list($data)) {
1303         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304             $res->{$1} = 1;
1305         } else {
1306             die "invalid hotplug feature '$feature'\n";
1307         }
1308     }
1309     return $res;
1310 }
1311
1312 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1313 sub pve_verify_hotplug_features {
1314     my ($value, $noerr) = @_;
1315
1316     return $value if parse_hotplug_features($value);
1317
1318     return if $noerr;
1319
1320     die "unable to parse hotplug option\n";
1321 }
1322
1323 sub scsi_inquiry {
1324     my($fh, $noerr) = @_;
1325
1326     my $SG_IO = 0x2285;
1327     my $SG_GET_VERSION_NUM = 0x2282;
1328
1329     my $versionbuf = "\x00" x 8;
1330     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1331     if (!$ret) {
1332         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333         return;
1334     }
1335     my $version = unpack("I", $versionbuf);
1336     if ($version < 30000) {
1337         die "scsi generic interface too old\n"  if !$noerr;
1338         return;
1339     }
1340
1341     my $buf = "\x00" x 36;
1342     my $sensebuf = "\x00" x 8;
1343     my $cmd = pack("C x3 C x1", 0x12, 36);
1344
1345     # see /usr/include/scsi/sg.h
1346     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1347
1348     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1349                       length($sensebuf), 0, length($buf), $buf,
1350                       $cmd, $sensebuf, 6000);
1351
1352     $ret = ioctl($fh, $SG_IO, $packet);
1353     if (!$ret) {
1354         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1355         return;
1356     }
1357
1358     my @res = unpack($sg_io_hdr_t, $packet);
1359     if ($res[17] || $res[18]) {
1360         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1361         return;
1362     }
1363
1364     my $res = {};
1365     (my $byte0, my $byte1, $res->{vendor},
1366      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1367
1368     $res->{removable} = $byte1 & 128 ? 1 : 0;
1369     $res->{type} = $byte0 & 31;
1370
1371     return $res;
1372 }
1373
1374 sub path_is_scsi {
1375     my ($path) = @_;
1376
1377     my $fh = IO::File->new("+<$path") || return;
1378     my $res = scsi_inquiry($fh, 1);
1379     close($fh);
1380
1381     return $res;
1382 }
1383
1384 sub print_tabletdevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1388
1389     # we use uhci for old VMs because tablet driver was buggy in older qemu
1390     my $usbbus;
1391     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1392         $usbbus = 'ehci';
1393     } else {
1394         $usbbus = 'uhci';
1395     }
1396
1397     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 }
1399
1400 sub print_keyboarddevice_full {
1401     my ($conf, $arch, $machine) = @_;
1402
1403     return if $arch ne 'aarch64';
1404
1405     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 }
1407
1408 my sub get_drive_id {
1409     my ($drive) = @_;
1410     return "$drive->{interface}$drive->{index}";
1411 }
1412
1413 sub print_drivedevice_full {
1414     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1415
1416     my $device = '';
1417     my $maxdev = 0;
1418
1419     my $drive_id = get_drive_id($drive);
1420     if ($drive->{interface} eq 'virtio') {
1421         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1422         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1423         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1424     } elsif ($drive->{interface} eq 'scsi') {
1425
1426         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1427         my $unit = $drive->{index} % $maxdev;
1428         my $devicetype = 'hd';
1429         my $path = '';
1430         if (drive_is_cdrom($drive)) {
1431             $devicetype = 'cd';
1432         } else {
1433             if ($drive->{file} =~ m|^/|) {
1434                 $path = $drive->{file};
1435                 if (my $info = path_is_scsi($path)) {
1436                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1437                         $devicetype = 'block';
1438                     } elsif ($info->{type} == 1) { # tape
1439                         $devicetype = 'generic';
1440                     }
1441                 }
1442             } else {
1443                  $path = PVE::Storage::path($storecfg, $drive->{file});
1444             }
1445
1446             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1447             my $version = extract_version($machine_type, kvm_user_version());
1448             if ($path =~ m/^iscsi\:\/\// &&
1449                !min_version($version, 4, 1)) {
1450                 $devicetype = 'generic';
1451             }
1452         }
1453
1454         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1455             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1456         } else {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1458                 .",lun=$drive->{index}";
1459         }
1460         $device .= ",drive=drive-$drive_id,id=$drive_id";
1461
1462         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1463             $device .= ",rotation_rate=1";
1464         }
1465         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1466
1467     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1468         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1469         my $controller = int($drive->{index} / $maxdev);
1470         my $unit = $drive->{index} % $maxdev;
1471         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1472
1473         $device = "ide-$devicetype";
1474         if ($drive->{interface} eq 'ide') {
1475             $device .= ",bus=ide.$controller,unit=$unit";
1476         } else {
1477             $device .= ",bus=ahci$controller.$unit";
1478         }
1479         $device .= ",drive=drive-$drive_id,id=$drive_id";
1480
1481         if ($devicetype eq 'hd') {
1482             if (my $model = $drive->{model}) {
1483                 $model = URI::Escape::uri_unescape($model);
1484                 $device .= ",model=$model";
1485             }
1486             if ($drive->{ssd}) {
1487                 $device .= ",rotation_rate=1";
1488             }
1489         }
1490         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1491     } elsif ($drive->{interface} eq 'usb') {
1492         die "implement me";
1493         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1494     } else {
1495         die "unsupported interface type";
1496     }
1497
1498     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1499
1500     if (my $serial = $drive->{serial}) {
1501         $serial = URI::Escape::uri_unescape($serial);
1502         $device .= ",serial=$serial";
1503     }
1504
1505
1506     return $device;
1507 }
1508
1509 sub get_initiator_name {
1510     my $initiator;
1511
1512     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1513     while (defined(my $line = <$fh>)) {
1514         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1515         $initiator = $1;
1516         last;
1517     }
1518     $fh->close();
1519
1520     return $initiator;
1521 }
1522
1523 sub print_drive_commandline_full {
1524     my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1525
1526     my $path;
1527     my $volid = $drive->{file};
1528     my $format = $drive->{format};
1529     my $drive_id = get_drive_id($drive);
1530
1531     if (drive_is_cdrom($drive)) {
1532         $path = get_iso_path($storecfg, $vmid, $volid);
1533         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1534     } else {
1535         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1536         if ($storeid) {
1537             $path = PVE::Storage::path($storecfg, $volid);
1538             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1539             $format //= qemu_img_format($scfg, $volname);
1540         } else {
1541             $path = $volid;
1542             $format //= "raw";
1543         }
1544    }
1545
1546    my $is_rbd = $path =~ m/^rbd:/;
1547
1548     my $opts = '';
1549     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1550     foreach my $o (@qemu_drive_options) {
1551         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1552     }
1553
1554     # snapshot only accepts on|off
1555     if (defined($drive->{snapshot})) {
1556         my $v = $drive->{snapshot} ? 'on' : 'off';
1557         $opts .= ",snapshot=$v";
1558     }
1559
1560     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1561         my ($dir, $qmpname) = @$type;
1562         if (my $v = $drive->{"mbps$dir"}) {
1563             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1564         }
1565         if (my $v = $drive->{"mbps${dir}_max"}) {
1566             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1567         }
1568         if (my $v = $drive->{"bps${dir}_max_length"}) {
1569             $opts .= ",throttling.bps$qmpname-max-length=$v";
1570         }
1571         if (my $v = $drive->{"iops${dir}"}) {
1572             $opts .= ",throttling.iops$qmpname=$v";
1573         }
1574         if (my $v = $drive->{"iops${dir}_max"}) {
1575             $opts .= ",throttling.iops$qmpname-max=$v";
1576         }
1577         if (my $v = $drive->{"iops${dir}_max_length"}) {
1578             $opts .= ",throttling.iops$qmpname-max-length=$v";
1579         }
1580     }
1581
1582     if ($pbs_name) {
1583         $format = "rbd" if $is_rbd;
1584         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1585             if !$format;
1586         $opts .= ",format=alloc-track,file.driver=$format";
1587     } elsif ($format) {
1588         $opts .= ",format=$format";
1589     }
1590
1591     my $cache_direct = 0;
1592
1593     if (my $cache = $drive->{cache}) {
1594         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1595     } elsif (!drive_is_cdrom($drive)) {
1596         $opts .= ",cache=none";
1597         $cache_direct = 1;
1598     }
1599
1600     # aio native works only with O_DIRECT
1601     if (!$drive->{aio}) {
1602         if($cache_direct) {
1603             $opts .= ",aio=native";
1604         } else {
1605             $opts .= ",aio=threads";
1606         }
1607     }
1608
1609     if (!drive_is_cdrom($drive)) {
1610         my $detectzeroes;
1611         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1612             $detectzeroes = 'off';
1613         } elsif ($drive->{discard}) {
1614             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1615         } else {
1616             # This used to be our default with discard not being specified:
1617             $detectzeroes = 'on';
1618         }
1619
1620         # note: 'detect-zeroes' works per blockdev and we want it to persist
1621         # after the alloc-track is removed, so put it on 'file' directly
1622         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1623         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1624     }
1625
1626     if ($pbs_name) {
1627         $opts .= ",backing=$pbs_name";
1628         $opts .= ",auto-remove=on";
1629     }
1630
1631     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1632     my $file_param = "file";
1633     if ($pbs_name) {
1634         # non-rbd drivers require the underlying file to be a seperate block
1635         # node, so add a second .file indirection
1636         $file_param .= ".file" if !$is_rbd;
1637         $file_param .= ".filename";
1638     }
1639     my $pathinfo = $path ? "$file_param=$path," : '';
1640
1641     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1642 }
1643
1644 sub print_pbs_blockdev {
1645     my ($pbs_conf, $pbs_name) = @_;
1646     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1647     $blockdev .= ",repository=$pbs_conf->{repository}";
1648     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1649     $blockdev .= ",archive=$pbs_conf->{archive}";
1650     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1651     return $blockdev;
1652 }
1653
1654 sub print_netdevice_full {
1655     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1656
1657     my $device = $net->{model};
1658     if ($net->{model} eq 'virtio') {
1659          $device = 'virtio-net-pci';
1660      };
1661
1662     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1663     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1664     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1665         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1666         # and out of each queue plus one config interrupt and control vector queue
1667         my $vectors = $net->{queues} * 2 + 2;
1668         $tmpstr .= ",vectors=$vectors,mq=on";
1669     }
1670     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1671
1672     if (my $mtu = $net->{mtu}) {
1673         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1674             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1675             if ($mtu == 1) {
1676                  $mtu = $bridge_mtu;
1677             } elsif ($mtu < 576) {
1678                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1679             } elsif ($mtu > $bridge_mtu) {
1680                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1681             }
1682             $tmpstr .= ",host_mtu=$mtu";
1683         } else {
1684             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1685         }
1686     }
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     if (is_native($arch)) {
1725         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1726     }
1727
1728     my $vmname = $conf->{name} || "vm$vmid";
1729
1730     my $netdev = "";
1731     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733     if ($net->{bridge}) {
1734         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1735             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1736     } else {
1737         $netdev = "type=user,id=$netid,hostname=$vmname";
1738     }
1739
1740     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1741
1742     return $netdev;
1743 }
1744
1745 my $vga_map = {
1746     'cirrus' => 'cirrus-vga',
1747     'std' => 'VGA',
1748     'vmware' => 'vmware-svga',
1749     'virtio' => 'virtio-vga',
1750 };
1751
1752 sub print_vga_device {
1753     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1754
1755     my $type = $vga_map->{$vga->{type}};
1756     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1757         $type = 'virtio-gpu';
1758     }
1759     my $vgamem_mb = $vga->{memory};
1760
1761     my $max_outputs = '';
1762     if ($qxlnum) {
1763         $type = $id ? 'qxl' : 'qxl-vga';
1764
1765         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1766             # set max outputs so linux can have up to 4 qxl displays with one device
1767             if (min_version($machine_version, 4, 1)) {
1768                 $max_outputs = ",max_outputs=4";
1769             }
1770         }
1771     }
1772
1773     die "no devicetype for $vga->{type}\n" if !$type;
1774
1775     my $memory = "";
1776     if ($vgamem_mb) {
1777         if ($vga->{type} eq 'virtio') {
1778             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1779             $memory = ",max_hostmem=$bytes";
1780         } elsif ($qxlnum) {
1781             # from https://www.spice-space.org/multiple-monitors.html
1782             $memory = ",vgamem_mb=$vga->{memory}";
1783             my $ram = $vgamem_mb * 4;
1784             my $vram = $vgamem_mb * 2;
1785             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1786         } else {
1787             $memory = ",vgamem_mb=$vga->{memory}";
1788         }
1789     } elsif ($qxlnum && $id) {
1790         $memory = ",ram_size=67108864,vram_size=33554432";
1791     }
1792
1793     my $edidoff = "";
1794     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1795         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1796     }
1797
1798     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1799     my $vgaid = "vga" . ($id // '');
1800     my $pciaddr;
1801
1802     if ($q35 && $vgaid eq 'vga') {
1803         # the first display uses pcie.0 bus on q35 machines
1804         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1805     } else {
1806         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1807     }
1808
1809     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1810 }
1811
1812 sub parse_number_sets {
1813     my ($set) = @_;
1814     my $res = [];
1815     foreach my $part (split(/;/, $set)) {
1816         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818             push @$res, [ $1, $2 ];
1819         } else {
1820             die "invalid range: $part\n";
1821         }
1822     }
1823     return $res;
1824 }
1825
1826 sub parse_numa {
1827     my ($data) = @_;
1828
1829     my $res = parse_property_string($numa_fmt, $data);
1830     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1831     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1832     return $res;
1833 }
1834
1835 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1836 sub parse_net {
1837     my ($data) = @_;
1838
1839     my $res = eval { parse_property_string($net_fmt, $data) };
1840     if ($@) {
1841         warn $@;
1842         return;
1843     }
1844     if (!defined($res->{macaddr})) {
1845         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1846         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1847     }
1848     return $res;
1849 }
1850
1851 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1852 sub parse_ipconfig {
1853     my ($data) = @_;
1854
1855     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1856     if ($@) {
1857         warn $@;
1858         return;
1859     }
1860
1861     if ($res->{gw} && !$res->{ip}) {
1862         warn 'gateway specified without specifying an IP address';
1863         return;
1864     }
1865     if ($res->{gw6} && !$res->{ip6}) {
1866         warn 'IPv6 gateway specified without specifying an IPv6 address';
1867         return;
1868     }
1869     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1870         warn 'gateway specified together with DHCP';
1871         return;
1872     }
1873     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1874         # gw6 + auto/dhcp
1875         warn "IPv6 gateway specified together with $res->{ip6} address";
1876         return;
1877     }
1878
1879     if (!$res->{ip} && !$res->{ip6}) {
1880         return { ip => 'dhcp', ip6 => 'dhcp' };
1881     }
1882
1883     return $res;
1884 }
1885
1886 sub print_net {
1887     my $net = shift;
1888
1889     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1890 }
1891
1892 sub add_random_macs {
1893     my ($settings) = @_;
1894
1895     foreach my $opt (keys %$settings) {
1896         next if $opt !~ m/^net(\d+)$/;
1897         my $net = parse_net($settings->{$opt});
1898         next if !$net;
1899         $settings->{$opt} = print_net($net);
1900     }
1901 }
1902
1903 sub vm_is_volid_owner {
1904     my ($storecfg, $vmid, $volid) = @_;
1905
1906     if ($volid !~  m|^/|) {
1907         my ($path, $owner);
1908         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1909         if ($owner && ($owner == $vmid)) {
1910             return 1;
1911         }
1912     }
1913
1914     return;
1915 }
1916
1917 sub vmconfig_register_unused_drive {
1918     my ($storecfg, $vmid, $conf, $drive) = @_;
1919
1920     if (drive_is_cloudinit($drive)) {
1921         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1922         warn $@ if $@;
1923     } elsif (!drive_is_cdrom($drive)) {
1924         my $volid = $drive->{file};
1925         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1926             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1927         }
1928     }
1929 }
1930
1931 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1932 my $smbios1_fmt = {
1933     uuid => {
1934         type => 'string',
1935         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1936         format_description => 'UUID',
1937         description => "Set SMBIOS1 UUID.",
1938         optional => 1,
1939     },
1940     version => {
1941         type => 'string',
1942         pattern => '[A-Za-z0-9+\/]+={0,2}',
1943         format_description => 'Base64 encoded string',
1944         description => "Set SMBIOS1 version.",
1945         optional => 1,
1946     },
1947     serial => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 serial number.",
1952         optional => 1,
1953     },
1954     manufacturer => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 manufacturer.",
1959         optional => 1,
1960     },
1961     product => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 product ID.",
1966         optional => 1,
1967     },
1968     sku => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 SKU string.",
1973         optional => 1,
1974     },
1975     family => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 family string.",
1980         optional => 1,
1981     },
1982     base64 => {
1983         type => 'boolean',
1984         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1985         optional => 1,
1986     },
1987 };
1988
1989 sub parse_smbios1 {
1990     my ($data) = @_;
1991
1992     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1993     warn $@ if $@;
1994     return $res;
1995 }
1996
1997 sub print_smbios1 {
1998     my ($smbios1) = @_;
1999     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2000 }
2001
2002 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2003
2004 sub parse_watchdog {
2005     my ($value) = @_;
2006
2007     return if !$value;
2008
2009     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2010     warn $@ if $@;
2011     return $res;
2012 }
2013
2014 sub parse_guest_agent {
2015     my ($conf) = @_;
2016
2017     return {} if !defined($conf->{agent});
2018
2019     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2020     warn $@ if $@;
2021
2022     # if the agent is disabled ignore the other potentially set properties
2023     return {} if !$res->{enabled};
2024     return $res;
2025 }
2026
2027 sub get_qga_key {
2028     my ($conf, $key) = @_;
2029     return undef if !defined($conf->{agent});
2030
2031     my $agent = parse_guest_agent($conf);
2032     return $agent->{$key};
2033 }
2034
2035 sub parse_vga {
2036     my ($value) = @_;
2037
2038     return {} if !$value;
2039     my $res = eval { parse_property_string($vga_fmt, $value) };
2040     warn $@ if $@;
2041     return $res;
2042 }
2043
2044 sub parse_rng {
2045     my ($value) = @_;
2046
2047     return if !$value;
2048
2049     my $res = eval { parse_property_string($rng_fmt, $value) };
2050     warn $@ if $@;
2051     return $res;
2052 }
2053
2054 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2055 sub verify_usb_device {
2056     my ($value, $noerr) = @_;
2057
2058     return $value if parse_usb_device($value);
2059
2060     return if $noerr;
2061
2062     die "unable to parse usb device\n";
2063 }
2064
2065 # add JSON properties for create and set function
2066 sub json_config_properties {
2067     my $prop = shift;
2068
2069     foreach my $opt (keys %$confdesc) {
2070         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2071             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2072         $prop->{$opt} = $confdesc->{$opt};
2073     }
2074
2075     return $prop;
2076 }
2077
2078 # return copy of $confdesc_cloudinit to generate documentation
2079 sub cloudinit_config_properties {
2080
2081     return dclone($confdesc_cloudinit);
2082 }
2083
2084 sub check_type {
2085     my ($key, $value) = @_;
2086
2087     die "unknown setting '$key'\n" if !$confdesc->{$key};
2088
2089     my $type = $confdesc->{$key}->{type};
2090
2091     if (!defined($value)) {
2092         die "got undefined value\n";
2093     }
2094
2095     if ($value =~ m/[\n\r]/) {
2096         die "property contains a line feed\n";
2097     }
2098
2099     if ($type eq 'boolean') {
2100         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2101         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2102         die "type check ('boolean') failed - got '$value'\n";
2103     } elsif ($type eq 'integer') {
2104         return int($1) if $value =~ m/^(\d+)$/;
2105         die "type check ('integer') failed - got '$value'\n";
2106     } elsif ($type eq 'number') {
2107         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2108         die "type check ('number') failed - got '$value'\n";
2109     } elsif ($type eq 'string') {
2110         if (my $fmt = $confdesc->{$key}->{format}) {
2111             PVE::JSONSchema::check_format($fmt, $value);
2112             return $value;
2113         }
2114         $value =~ s/^\"(.*)\"$/$1/;
2115         return $value;
2116     } else {
2117         die "internal error"
2118     }
2119 }
2120
2121 sub destroy_vm {
2122     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2123
2124     my $conf = PVE::QemuConfig->load_config($vmid);
2125
2126     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2127
2128     if ($conf->{template}) {
2129         # check if any base image is still used by a linked clone
2130         PVE::QemuConfig->foreach_volume($conf, sub {
2131                 my ($ds, $drive) = @_;
2132                 return if drive_is_cdrom($drive);
2133
2134                 my $volid = $drive->{file};
2135                 return if !$volid || $volid =~ m|^/|;
2136
2137                 die "base volume '$volid' is still in use by linked cloned\n"
2138                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2139
2140         });
2141     }
2142
2143     # only remove disks owned by this VM (referenced in the config)
2144     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2145         my ($ds, $drive) = @_;
2146         return if drive_is_cdrom($drive, 1);
2147
2148         my $volid = $drive->{file};
2149         return if !$volid || $volid =~ m|^/|;
2150
2151         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2152         return if !$path || !$owner || ($owner != $vmid);
2153
2154         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2155         warn "Could not remove disk '$volid', check manually: $@" if $@;
2156     });
2157
2158     if ($purge_unreferenced) { # also remove unreferenced disk
2159         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2160         PVE::Storage::foreach_volid($vmdisks, sub {
2161             my ($volid, $sid, $volname, $d) = @_;
2162             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2163             warn $@ if $@;
2164         });
2165     }
2166
2167     if (defined $replacement_conf) {
2168         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2169     } else {
2170         PVE::QemuConfig->destroy_config($vmid);
2171     }
2172 }
2173
2174 sub parse_vm_config {
2175     my ($filename, $raw) = @_;
2176
2177     return if !defined($raw);
2178
2179     my $res = {
2180         digest => Digest::SHA::sha1_hex($raw),
2181         snapshots => {},
2182         pending => {},
2183     };
2184
2185     $filename =~ m|/qemu-server/(\d+)\.conf$|
2186         || die "got strange filename '$filename'";
2187
2188     my $vmid = $1;
2189
2190     my $conf = $res;
2191     my $descr;
2192     my $section = '';
2193
2194     my @lines = split(/\n/, $raw);
2195     foreach my $line (@lines) {
2196         next if $line =~ m/^\s*$/;
2197
2198         if ($line =~ m/^\[PENDING\]\s*$/i) {
2199             $section = 'pending';
2200             if (defined($descr)) {
2201                 $descr =~ s/\s+$//;
2202                 $conf->{description} = $descr;
2203             }
2204             $descr = undef;
2205             $conf = $res->{$section} = {};
2206             next;
2207
2208         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2209             $section = $1;
2210             if (defined($descr)) {
2211                 $descr =~ s/\s+$//;
2212                 $conf->{description} = $descr;
2213             }
2214             $descr = undef;
2215             $conf = $res->{snapshots}->{$section} = {};
2216             next;
2217         }
2218
2219         if ($line =~ m/^\#(.*)\s*$/) {
2220             $descr = '' if !defined($descr);
2221             $descr .= PVE::Tools::decode_text($1) . "\n";
2222             next;
2223         }
2224
2225         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2226             $descr = '' if !defined($descr);
2227             $descr .= PVE::Tools::decode_text($2);
2228         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2229             $conf->{snapstate} = $1;
2230         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2231             my $key = $1;
2232             my $value = $2;
2233             $conf->{$key} = $value;
2234         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2235             my $value = $1;
2236             if ($section eq 'pending') {
2237                 $conf->{delete} = $value; # we parse this later
2238             } else {
2239                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2240             }
2241         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2242             my $key = $1;
2243             my $value = $2;
2244             eval { $value = check_type($key, $value); };
2245             if ($@) {
2246                 warn "vm $vmid - unable to parse value of '$key' - $@";
2247             } else {
2248                 $key = 'ide2' if $key eq 'cdrom';
2249                 my $fmt = $confdesc->{$key}->{format};
2250                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2251                     my $v = parse_drive($key, $value);
2252                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2253                         $v->{file} = $volid;
2254                         $value = print_drive($v);
2255                     } else {
2256                         warn "vm $vmid - unable to parse value of '$key'\n";
2257                         next;
2258                     }
2259                 }
2260
2261                 $conf->{$key} = $value;
2262             }
2263         } else {
2264             warn "vm $vmid - unable to parse config: $line\n";
2265         }
2266     }
2267
2268     if (defined($descr)) {
2269         $descr =~ s/\s+$//;
2270         $conf->{description} = $descr;
2271     }
2272     delete $res->{snapstate}; # just to be sure
2273
2274     return $res;
2275 }
2276
2277 sub write_vm_config {
2278     my ($filename, $conf) = @_;
2279
2280     delete $conf->{snapstate}; # just to be sure
2281
2282     if ($conf->{cdrom}) {
2283         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2284         $conf->{ide2} = $conf->{cdrom};
2285         delete $conf->{cdrom};
2286     }
2287
2288     # we do not use 'smp' any longer
2289     if ($conf->{sockets}) {
2290         delete $conf->{smp};
2291     } elsif ($conf->{smp}) {
2292         $conf->{sockets} = $conf->{smp};
2293         delete $conf->{cores};
2294         delete $conf->{smp};
2295     }
2296
2297     my $used_volids = {};
2298
2299     my $cleanup_config = sub {
2300         my ($cref, $pending, $snapname) = @_;
2301
2302         foreach my $key (keys %$cref) {
2303             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2304                 $key eq 'snapstate' || $key eq 'pending';
2305             my $value = $cref->{$key};
2306             if ($key eq 'delete') {
2307                 die "propertry 'delete' is only allowed in [PENDING]\n"
2308                     if !$pending;
2309                 # fixme: check syntax?
2310                 next;
2311             }
2312             eval { $value = check_type($key, $value); };
2313             die "unable to parse value of '$key' - $@" if $@;
2314
2315             $cref->{$key} = $value;
2316
2317             if (!$snapname && is_valid_drivename($key)) {
2318                 my $drive = parse_drive($key, $value);
2319                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2320             }
2321         }
2322     };
2323
2324     &$cleanup_config($conf);
2325
2326     &$cleanup_config($conf->{pending}, 1);
2327
2328     foreach my $snapname (keys %{$conf->{snapshots}}) {
2329         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2330         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2331     }
2332
2333     # remove 'unusedX' settings if we re-add a volume
2334     foreach my $key (keys %$conf) {
2335         my $value = $conf->{$key};
2336         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2337             delete $conf->{$key};
2338         }
2339     }
2340
2341     my $generate_raw_config = sub {
2342         my ($conf, $pending) = @_;
2343
2344         my $raw = '';
2345
2346         # add description as comment to top of file
2347         if (defined(my $descr = $conf->{description})) {
2348             if ($descr) {
2349                 foreach my $cl (split(/\n/, $descr)) {
2350                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2351                 }
2352             } else {
2353                 $raw .= "#\n" if $pending;
2354             }
2355         }
2356
2357         foreach my $key (sort keys %$conf) {
2358             next if $key =~ /^(digest|description|pending|snapshots)$/;
2359             $raw .= "$key: $conf->{$key}\n";
2360         }
2361         return $raw;
2362     };
2363
2364     my $raw = &$generate_raw_config($conf);
2365
2366     if (scalar(keys %{$conf->{pending}})){
2367         $raw .= "\n[PENDING]\n";
2368         $raw .= &$generate_raw_config($conf->{pending}, 1);
2369     }
2370
2371     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2372         $raw .= "\n[$snapname]\n";
2373         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2374     }
2375
2376     return $raw;
2377 }
2378
2379 sub load_defaults {
2380
2381     my $res = {};
2382
2383     # we use static defaults from our JSON schema configuration
2384     foreach my $key (keys %$confdesc) {
2385         if (defined(my $default = $confdesc->{$key}->{default})) {
2386             $res->{$key} = $default;
2387         }
2388     }
2389
2390     return $res;
2391 }
2392
2393 sub config_list {
2394     my $vmlist = PVE::Cluster::get_vmlist();
2395     my $res = {};
2396     return $res if !$vmlist || !$vmlist->{ids};
2397     my $ids = $vmlist->{ids};
2398     my $nodename = nodename();
2399
2400     foreach my $vmid (keys %$ids) {
2401         my $d = $ids->{$vmid};
2402         next if !$d->{node} || $d->{node} ne $nodename;
2403         next if !$d->{type} || $d->{type} ne 'qemu';
2404         $res->{$vmid}->{exists} = 1;
2405     }
2406     return $res;
2407 }
2408
2409 # test if VM uses local resources (to prevent migration)
2410 sub check_local_resources {
2411     my ($conf, $noerr) = @_;
2412
2413     my @loc_res = ();
2414
2415     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2416     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2417
2418     push @loc_res, "ivshmem" if $conf->{ivshmem};
2419
2420     foreach my $k (keys %$conf) {
2421         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2422         # sockets are safe: they will recreated be on the target side post-migrate
2423         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2424         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2425     }
2426
2427     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2428
2429     return \@loc_res;
2430 }
2431
2432 # check if used storages are available on all nodes (use by migrate)
2433 sub check_storage_availability {
2434     my ($storecfg, $conf, $node) = @_;
2435
2436     PVE::QemuConfig->foreach_volume($conf, sub {
2437         my ($ds, $drive) = @_;
2438
2439         my $volid = $drive->{file};
2440         return if !$volid;
2441
2442         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2443         return if !$sid;
2444
2445         # check if storage is available on both nodes
2446         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2447         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2448    });
2449 }
2450
2451 # list nodes where all VM images are available (used by has_feature API)
2452 sub shared_nodes {
2453     my ($conf, $storecfg) = @_;
2454
2455     my $nodelist = PVE::Cluster::get_nodelist();
2456     my $nodehash = { map { $_ => 1 } @$nodelist };
2457     my $nodename = nodename();
2458
2459     PVE::QemuConfig->foreach_volume($conf, sub {
2460         my ($ds, $drive) = @_;
2461
2462         my $volid = $drive->{file};
2463         return if !$volid;
2464
2465         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2466         if ($storeid) {
2467             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2468             if ($scfg->{disable}) {
2469                 $nodehash = {};
2470             } elsif (my $avail = $scfg->{nodes}) {
2471                 foreach my $node (keys %$nodehash) {
2472                     delete $nodehash->{$node} if !$avail->{$node};
2473                 }
2474             } elsif (!$scfg->{shared}) {
2475                 foreach my $node (keys %$nodehash) {
2476                     delete $nodehash->{$node} if $node ne $nodename
2477                 }
2478             }
2479         }
2480     });
2481
2482     return $nodehash
2483 }
2484
2485 sub check_local_storage_availability {
2486     my ($conf, $storecfg) = @_;
2487
2488     my $nodelist = PVE::Cluster::get_nodelist();
2489     my $nodehash = { map { $_ => {} } @$nodelist };
2490
2491     PVE::QemuConfig->foreach_volume($conf, sub {
2492         my ($ds, $drive) = @_;
2493
2494         my $volid = $drive->{file};
2495         return if !$volid;
2496
2497         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2498         if ($storeid) {
2499             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2500
2501             if ($scfg->{disable}) {
2502                 foreach my $node (keys %$nodehash) {
2503                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2504                 }
2505             } elsif (my $avail = $scfg->{nodes}) {
2506                 foreach my $node (keys %$nodehash) {
2507                     if (!$avail->{$node}) {
2508                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2509                     }
2510                 }
2511             }
2512         }
2513     });
2514
2515     foreach my $node (values %$nodehash) {
2516         if (my $unavail = $node->{unavailable_storages}) {
2517             $node->{unavailable_storages} = [ sort keys %$unavail ];
2518         }
2519     }
2520
2521     return $nodehash
2522 }
2523
2524 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2525 sub check_running {
2526     my ($vmid, $nocheck, $node) = @_;
2527
2528     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2529     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2530 }
2531
2532 sub vzlist {
2533
2534     my $vzlist = config_list();
2535
2536     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2537
2538     while (defined(my $de = $fd->read)) {
2539         next if $de !~ m/^(\d+)\.pid$/;
2540         my $vmid = $1;
2541         next if !defined($vzlist->{$vmid});
2542         if (my $pid = check_running($vmid)) {
2543             $vzlist->{$vmid}->{pid} = $pid;
2544         }
2545     }
2546
2547     return $vzlist;
2548 }
2549
2550 our $vmstatus_return_properties = {
2551     vmid => get_standard_option('pve-vmid'),
2552     status => {
2553         description => "Qemu process status.",
2554         type => 'string',
2555         enum => ['stopped', 'running'],
2556     },
2557     maxmem => {
2558         description => "Maximum memory in bytes.",
2559         type => 'integer',
2560         optional => 1,
2561         renderer => 'bytes',
2562     },
2563     maxdisk => {
2564         description => "Root disk size in bytes.",
2565         type => 'integer',
2566         optional => 1,
2567         renderer => 'bytes',
2568     },
2569     name => {
2570         description => "VM name.",
2571         type => 'string',
2572         optional => 1,
2573     },
2574     qmpstatus => {
2575         description => "Qemu QMP agent status.",
2576         type => 'string',
2577         optional => 1,
2578     },
2579     pid => {
2580         description => "PID of running qemu process.",
2581         type => 'integer',
2582         optional => 1,
2583     },
2584     uptime => {
2585         description => "Uptime.",
2586         type => 'integer',
2587         optional => 1,
2588         renderer => 'duration',
2589     },
2590     cpus => {
2591         description => "Maximum usable CPUs.",
2592         type => 'number',
2593         optional => 1,
2594     },
2595     lock => {
2596         description => "The current config lock, if any.",
2597         type => 'string',
2598         optional => 1,
2599     },
2600     tags => {
2601         description  => "The current configured tags, if any",
2602         type => 'string',
2603         optional => 1,
2604     },
2605     'running-machine' => {
2606         description => "The currently running machine type (if running).",
2607         type => 'string',
2608         optional => 1,
2609     },
2610     'running-qemu' => {
2611         description => "The currently running QEMU version (if running).",
2612         type => 'string',
2613         optional => 1,
2614     },
2615 };
2616
2617 my $last_proc_pid_stat;
2618
2619 # get VM status information
2620 # This must be fast and should not block ($full == false)
2621 # We only query KVM using QMP if $full == true (this can be slow)
2622 sub vmstatus {
2623     my ($opt_vmid, $full) = @_;
2624
2625     my $res = {};
2626
2627     my $storecfg = PVE::Storage::config();
2628
2629     my $list = vzlist();
2630     my $defaults = load_defaults();
2631
2632     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2633
2634     my $cpucount = $cpuinfo->{cpus} || 1;
2635
2636     foreach my $vmid (keys %$list) {
2637         next if $opt_vmid && ($vmid ne $opt_vmid);
2638
2639         my $conf = PVE::QemuConfig->load_config($vmid);
2640
2641         my $d = { vmid => $vmid };
2642         $d->{pid} = $list->{$vmid}->{pid};
2643
2644         # fixme: better status?
2645         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2646
2647         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2648         if (defined($size)) {
2649             $d->{disk} = 0; # no info available
2650             $d->{maxdisk} = $size;
2651         } else {
2652             $d->{disk} = 0;
2653             $d->{maxdisk} = 0;
2654         }
2655
2656         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2657             * ($conf->{cores} || $defaults->{cores});
2658         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2659         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2660
2661         $d->{name} = $conf->{name} || "VM $vmid";
2662         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2663             : $defaults->{memory}*(1024*1024);
2664
2665         if ($conf->{balloon}) {
2666             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2667             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2668                 : $defaults->{shares};
2669         }
2670
2671         $d->{uptime} = 0;
2672         $d->{cpu} = 0;
2673         $d->{mem} = 0;
2674
2675         $d->{netout} = 0;
2676         $d->{netin} = 0;
2677
2678         $d->{diskread} = 0;
2679         $d->{diskwrite} = 0;
2680
2681         $d->{template} = PVE::QemuConfig->is_template($conf);
2682
2683         $d->{serial} = 1 if conf_has_serial($conf);
2684         $d->{lock} = $conf->{lock} if $conf->{lock};
2685         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2686
2687         $res->{$vmid} = $d;
2688     }
2689
2690     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2691     foreach my $dev (keys %$netdev) {
2692         next if $dev !~ m/^tap([1-9]\d*)i/;
2693         my $vmid = $1;
2694         my $d = $res->{$vmid};
2695         next if !$d;
2696
2697         $d->{netout} += $netdev->{$dev}->{receive};
2698         $d->{netin} += $netdev->{$dev}->{transmit};
2699
2700         if ($full) {
2701             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2702             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2703         }
2704
2705     }
2706
2707     my $ctime = gettimeofday;
2708
2709     foreach my $vmid (keys %$list) {
2710
2711         my $d = $res->{$vmid};
2712         my $pid = $d->{pid};
2713         next if !$pid;
2714
2715         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2716         next if !$pstat; # not running
2717
2718         my $used = $pstat->{utime} + $pstat->{stime};
2719
2720         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2721
2722         if ($pstat->{vsize}) {
2723             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2724         }
2725
2726         my $old = $last_proc_pid_stat->{$pid};
2727         if (!$old) {
2728             $last_proc_pid_stat->{$pid} = {
2729                 time => $ctime,
2730                 used => $used,
2731                 cpu => 0,
2732             };
2733             next;
2734         }
2735
2736         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2737
2738         if ($dtime > 1000) {
2739             my $dutime = $used -  $old->{used};
2740
2741             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2742             $last_proc_pid_stat->{$pid} = {
2743                 time => $ctime,
2744                 used => $used,
2745                 cpu => $d->{cpu},
2746             };
2747         } else {
2748             $d->{cpu} = $old->{cpu};
2749         }
2750     }
2751
2752     return $res if !$full;
2753
2754     my $qmpclient = PVE::QMPClient->new();
2755
2756     my $ballooncb = sub {
2757         my ($vmid, $resp) = @_;
2758
2759         my $info = $resp->{'return'};
2760         return if !$info->{max_mem};
2761
2762         my $d = $res->{$vmid};
2763
2764         # use memory assigned to VM
2765         $d->{maxmem} = $info->{max_mem};
2766         $d->{balloon} = $info->{actual};
2767
2768         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2769             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2770             $d->{freemem} = $info->{free_mem};
2771         }
2772
2773         $d->{ballooninfo} = $info;
2774     };
2775
2776     my $blockstatscb = sub {
2777         my ($vmid, $resp) = @_;
2778         my $data = $resp->{'return'} || [];
2779         my $totalrdbytes = 0;
2780         my $totalwrbytes = 0;
2781
2782         for my $blockstat (@$data) {
2783             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2784             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2785
2786             $blockstat->{device} =~ s/drive-//;
2787             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2788         }
2789         $res->{$vmid}->{diskread} = $totalrdbytes;
2790         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2791     };
2792
2793     my $machinecb = sub {
2794         my ($vmid, $resp) = @_;
2795         my $data = $resp->{'return'} || [];
2796
2797         $res->{$vmid}->{'running-machine'} =
2798             PVE::QemuServer::Machine::current_from_query_machines($data);
2799     };
2800
2801     my $versioncb = sub {
2802         my ($vmid, $resp) = @_;
2803         my $data = $resp->{'return'} // {};
2804         my $version = 'unknown';
2805
2806         if (my $v = $data->{qemu}) {
2807             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2808         }
2809
2810         $res->{$vmid}->{'running-qemu'} = $version;
2811     };
2812
2813     my $statuscb = sub {
2814         my ($vmid, $resp) = @_;
2815
2816         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2817         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2818         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2819         # this fails if ballon driver is not loaded, so this must be
2820         # the last commnand (following command are aborted if this fails).
2821         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2822
2823         my $status = 'unknown';
2824         if (!defined($status = $resp->{'return'}->{status})) {
2825             warn "unable to get VM status\n";
2826             return;
2827         }
2828
2829         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2830     };
2831
2832     foreach my $vmid (keys %$list) {
2833         next if $opt_vmid && ($vmid ne $opt_vmid);
2834         next if !$res->{$vmid}->{pid}; # not running
2835         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2836     }
2837
2838     $qmpclient->queue_execute(undef, 2);
2839
2840     foreach my $vmid (keys %$list) {
2841         next if $opt_vmid && ($vmid ne $opt_vmid);
2842         next if !$res->{$vmid}->{pid}; #not running
2843
2844         # we can't use the $qmpclient since it might have already aborted on
2845         # 'query-balloon', but this might also fail for older versions...
2846         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2847         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2848     }
2849
2850     foreach my $vmid (keys %$list) {
2851         next if $opt_vmid && ($vmid ne $opt_vmid);
2852         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2853     }
2854
2855     return $res;
2856 }
2857
2858 sub conf_has_serial {
2859     my ($conf) = @_;
2860
2861     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2862         if ($conf->{"serial$i"}) {
2863             return 1;
2864         }
2865     }
2866
2867     return 0;
2868 }
2869
2870 sub conf_has_audio {
2871     my ($conf, $id) = @_;
2872
2873     $id //= 0;
2874     my $audio = $conf->{"audio$id"};
2875     return if !defined($audio);
2876
2877     my $audioproperties = parse_property_string($audio_fmt, $audio);
2878     my $audiodriver = $audioproperties->{driver} // 'spice';
2879
2880     return {
2881         dev => $audioproperties->{device},
2882         dev_id => "audiodev$id",
2883         backend => $audiodriver,
2884         backend_id => "$audiodriver-backend${id}",
2885     };
2886 }
2887
2888 sub audio_devs {
2889     my ($audio, $audiopciaddr, $machine_version) = @_;
2890
2891     my $devs = [];
2892
2893     my $id = $audio->{dev_id};
2894     my $audiodev = "";
2895     if (min_version($machine_version, 4, 2)) {
2896         $audiodev = ",audiodev=$audio->{backend_id}";
2897     }
2898
2899     if ($audio->{dev} eq 'AC97') {
2900         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2901     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2902         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2903         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2904         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2905     } else {
2906         die "unkown audio device '$audio->{dev}', implement me!";
2907     }
2908
2909     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2910
2911     return $devs;
2912 }
2913
2914 sub vga_conf_has_spice {
2915     my ($vga) = @_;
2916
2917     my $vgaconf = parse_vga($vga);
2918     my $vgatype = $vgaconf->{type};
2919     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2920
2921     return $1 || 1;
2922 }
2923
2924 sub is_native($) {
2925     my ($arch) = @_;
2926     return get_host_arch() eq $arch;
2927 }
2928
2929 sub get_vm_arch {
2930     my ($conf) = @_;
2931     return $conf->{arch} // get_host_arch();
2932 }
2933
2934 my $default_machines = {
2935     x86_64 => 'pc',
2936     aarch64 => 'virt',
2937 };
2938
2939 sub get_installed_machine_version {
2940     my ($kvmversion) = @_;
2941     $kvmversion = kvm_user_version() if !defined($kvmversion);
2942     $kvmversion =~ m/^(\d+\.\d+)/;
2943     return $1;
2944 }
2945
2946 sub windows_get_pinned_machine_version {
2947     my ($machine, $base_version, $kvmversion) = @_;
2948
2949     my $pin_version = $base_version;
2950     if (!defined($base_version) ||
2951         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2952     ) {
2953         $pin_version = get_installed_machine_version($kvmversion);
2954     }
2955     if (!$machine || $machine eq 'pc') {
2956         $machine = "pc-i440fx-$pin_version";
2957     } elsif ($machine eq 'q35') {
2958         $machine = "pc-q35-$pin_version";
2959     } elsif ($machine eq 'virt') {
2960         $machine = "virt-$pin_version";
2961     } else {
2962         warn "unknown machine type '$machine', not touching that!\n";
2963     }
2964
2965     return $machine;
2966 }
2967
2968 sub get_vm_machine {
2969     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2970
2971     my $machine = $forcemachine || $conf->{machine};
2972
2973     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2974         $kvmversion //= kvm_user_version();
2975         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2976         # layout which confuses windows quite a bit and may result in various regressions..
2977         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2978         if (windows_version($conf->{ostype})) {
2979             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2980         }
2981         $arch //= 'x86_64';
2982         $machine ||= $default_machines->{$arch};
2983         if ($add_pve_version) {
2984             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2985             $machine .= "+pve$pvever";
2986         }
2987     }
2988
2989     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2990         # for version-pinned machines that do not include a pve-version (e.g.
2991         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2992         $machine .= '+pve0';
2993     }
2994
2995     return $machine;
2996 }
2997
2998 sub get_ovmf_files($) {
2999     my ($arch) = @_;
3000
3001     my $ovmf = $OVMF->{$arch}
3002         or die "no OVMF images known for architecture '$arch'\n";
3003
3004     return @$ovmf;
3005 }
3006
3007 my $Arch2Qemu = {
3008     aarch64 => '/usr/bin/qemu-system-aarch64',
3009     x86_64 => '/usr/bin/qemu-system-x86_64',
3010 };
3011 sub get_command_for_arch($) {
3012     my ($arch) = @_;
3013     return '/usr/bin/kvm' if is_native($arch);
3014
3015     my $cmd = $Arch2Qemu->{$arch}
3016         or die "don't know how to emulate architecture '$arch'\n";
3017     return $cmd;
3018 }
3019
3020 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3021 # to use in a QEMU command line (-cpu element), first array_intersect the result
3022 # of query_supported_ with query_understood_. This is necessary because:
3023 #
3024 # a) query_understood_ returns flags the host cannot use and
3025 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3026 #    flags, but CPU settings - with most of them being flags. Those settings
3027 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3028 #
3029 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3030 # expensive. If you need the value returned from this, you can get it much
3031 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3032 # $accel being 'kvm' or 'tcg'.
3033 #
3034 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3035 # changes, automatically populating pmxcfs.
3036 #
3037 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3038 # since kvm and tcg machines support different flags
3039 #
3040 sub query_supported_cpu_flags {
3041     my ($arch) = @_;
3042
3043     $arch //= get_host_arch();
3044     my $default_machine = $default_machines->{$arch};
3045
3046     my $flags = {};
3047
3048     # FIXME: Once this is merged, the code below should work for ARM as well:
3049     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3050     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3051         $arch eq "aarch64";
3052
3053     my $kvm_supported = defined(kvm_version());
3054     my $qemu_cmd = get_command_for_arch($arch);
3055     my $fakevmid = -1;
3056     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3057
3058     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3059     my $query_supported_run_qemu = sub {
3060         my ($kvm) = @_;
3061
3062         my $flags = {};
3063         my $cmd = [
3064             $qemu_cmd,
3065             '-machine', $default_machine,
3066             '-display', 'none',
3067             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3068             '-mon', 'chardev=qmp,mode=control',
3069             '-pidfile', $pidfile,
3070             '-S', '-daemonize'
3071         ];
3072
3073         if (!$kvm) {
3074             push @$cmd, '-accel', 'tcg';
3075         }
3076
3077         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3078         die "QEMU flag querying VM exited with code " . $rc if $rc;
3079
3080         eval {
3081             my $cmd_result = mon_cmd(
3082                 $fakevmid,
3083                 'query-cpu-model-expansion',
3084                 type => 'full',
3085                 model => { name => 'host' }
3086             );
3087
3088             my $props = $cmd_result->{model}->{props};
3089             foreach my $prop (keys %$props) {
3090                 next if $props->{$prop} ne '1';
3091                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3092                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3093                 # We only keep those with underscores, to match /proc/cpuinfo
3094                 $prop =~ s/\.|-/_/g;
3095                 $flags->{$prop} = 1;
3096             }
3097         };
3098         my $err = $@;
3099
3100         # force stop with 10 sec timeout and 'nocheck'
3101         # always stop, even if QMP failed
3102         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3103
3104         die $err if $err;
3105
3106         return [ sort keys %$flags ];
3107     };
3108
3109     # We need to query QEMU twice, since KVM and TCG have different supported flags
3110     PVE::QemuConfig->lock_config($fakevmid, sub {
3111         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3112         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3113
3114         if ($kvm_supported) {
3115             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3116             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3117         }
3118     });
3119
3120     return $flags;
3121 }
3122
3123 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3124 my $understood_cpu_flag_dir = "/usr/share/kvm";
3125 sub query_understood_cpu_flags {
3126     my $arch = get_host_arch();
3127     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3128
3129     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3130         if ! -e $filepath;
3131
3132     my $raw = file_get_contents($filepath);
3133     $raw =~ s/^\s+|\s+$//g;
3134     my @flags = split(/\s+/, $raw);
3135
3136     return \@flags;
3137 }
3138
3139 sub config_to_command {
3140     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3141         $pbs_backing) = @_;
3142
3143     my $cmd = [];
3144     my $globalFlags = [];
3145     my $machineFlags = [];
3146     my $rtcFlags = [];
3147     my $devices = [];
3148     my $pciaddr = '';
3149     my $bridges = {};
3150     my $ostype = $conf->{ostype};
3151     my $winversion = windows_version($ostype);
3152     my $kvm = $conf->{kvm};
3153     my $nodename = nodename();
3154
3155     my $arch = get_vm_arch($conf);
3156     my $kvm_binary = get_command_for_arch($arch);
3157     my $kvmver = kvm_user_version($kvm_binary);
3158
3159     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3160         $kvmver //= "undefined";
3161         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3162     }
3163
3164     my $add_pve_version = min_version($kvmver, 4, 1);
3165
3166     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3167     my $machine_version = extract_version($machine_type, $kvmver);
3168     $kvm //= 1 if is_native($arch);
3169
3170     $machine_version =~ m/(\d+)\.(\d+)/;
3171     my ($machine_major, $machine_minor) = ($1, $2);
3172
3173     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3174         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3175     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3176         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3177             ." please upgrade node '$nodename'\n"
3178     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3179         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3180         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3181             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3182             ." node '$nodename'\n";
3183     }
3184
3185     # if a specific +pve version is required for a feature, use $version_guard
3186     # instead of min_version to allow machines to be run with the minimum
3187     # required version
3188     my $required_pve_version = 0;
3189     my $version_guard = sub {
3190         my ($major, $minor, $pve) = @_;
3191         return 0 if !min_version($machine_version, $major, $minor, $pve);
3192         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3193         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3194         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3195         return 1;
3196     };
3197
3198     if ($kvm && !defined kvm_version()) {
3199         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3200             ." or enable in BIOS.\n";
3201     }
3202
3203     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3204     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3205     my $use_old_bios_files = undef;
3206     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3207
3208     my $cpuunits = defined($conf->{cpuunits}) ?
3209             $conf->{cpuunits} : $defaults->{cpuunits};
3210
3211     push @$cmd, $kvm_binary;
3212
3213     push @$cmd, '-id', $vmid;
3214
3215     my $vmname = $conf->{name} || "vm$vmid";
3216
3217     push @$cmd, '-name', $vmname;
3218
3219     push @$cmd, '-no-shutdown';
3220
3221     my $use_virtio = 0;
3222
3223     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3224     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3225     push @$cmd, '-mon', "chardev=qmp,mode=control";
3226
3227     if (min_version($machine_version, 2, 12)) {
3228         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3229         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3230     }
3231
3232     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3233
3234     push @$cmd, '-daemonize';
3235
3236     if ($conf->{smbios1}) {
3237         my $smbios_conf = parse_smbios1($conf->{smbios1});
3238         if ($smbios_conf->{base64}) {
3239             # Do not pass base64 flag to qemu
3240             delete $smbios_conf->{base64};
3241             my $smbios_string = "";
3242             foreach my $key (keys %$smbios_conf) {
3243                 my $value;
3244                 if ($key eq "uuid") {
3245                     $value = $smbios_conf->{uuid}
3246                 } else {
3247                     $value = decode_base64($smbios_conf->{$key});
3248                 }
3249                 # qemu accepts any binary data, only commas need escaping by double comma
3250                 $value =~ s/,/,,/g;
3251                 $smbios_string .= "," . $key . "=" . $value if $value;
3252             }
3253             push @$cmd, '-smbios', "type=1" . $smbios_string;
3254         } else {
3255             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3256         }
3257     }
3258
3259     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3260         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3261         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3262
3263         my ($path, $format);
3264         if (my $efidisk = $conf->{efidisk0}) {
3265             my $d = parse_drive('efidisk0', $efidisk);
3266             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3267             $format = $d->{format};
3268             if ($storeid) {
3269                 $path = PVE::Storage::path($storecfg, $d->{file});
3270                 if (!defined($format)) {
3271                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3272                     $format = qemu_img_format($scfg, $volname);
3273                 }
3274             } else {
3275                 $path = $d->{file};
3276                 die "efidisk format must be specified\n"
3277                     if !defined($format);
3278             }
3279         } else {
3280             warn "no efidisk configured! Using temporary efivars disk.\n";
3281             $path = "/tmp/$vmid-ovmf.fd";
3282             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3283             $format = 'raw';
3284         }
3285
3286         my $size_str = "";
3287
3288         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3289             $size_str = ",size=" . (-s $ovmf_vars);
3290         }
3291
3292         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3293         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3294     }
3295
3296     # load q35 config
3297     if ($q35) {
3298         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3299         if (min_version($machine_version, 4, 0)) {
3300             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3301         } else {
3302             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3303         }
3304     }
3305
3306     if ($conf->{vmgenid}) {
3307         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3308     }
3309
3310     # add usb controllers
3311     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3312         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3313     push @$devices, @usbcontrollers if @usbcontrollers;
3314     my $vga = parse_vga($conf->{vga});
3315
3316     my $qxlnum = vga_conf_has_spice($conf->{vga});
3317     $vga->{type} = 'qxl' if $qxlnum;
3318
3319     if (!$vga->{type}) {
3320         if ($arch eq 'aarch64') {
3321             $vga->{type} = 'virtio';
3322         } elsif (min_version($machine_version, 2, 9)) {
3323             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3324         } else {
3325             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3326         }
3327     }
3328
3329     # enable absolute mouse coordinates (needed by vnc)
3330     my $tablet;
3331     if (defined($conf->{tablet})) {
3332         $tablet = $conf->{tablet};
3333     } else {
3334         $tablet = $defaults->{tablet};
3335         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3336         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3337     }
3338
3339     if ($tablet) {
3340         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3341         my $kbd = print_keyboarddevice_full($conf, $arch);
3342         push @$devices, '-device', $kbd if defined($kbd);
3343     }
3344
3345     my $bootorder = device_bootorder($conf);
3346
3347     # host pci device passthrough
3348     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3349         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3350
3351     # usb devices
3352     my $usb_dev_features = {};
3353     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3354
3355     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3356         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3357     push @$devices, @usbdevices if @usbdevices;
3358
3359     # serial devices
3360     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3361         if (my $path = $conf->{"serial$i"}) {
3362             if ($path eq 'socket') {
3363                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3364                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3365                 # On aarch64, serial0 is the UART device. Qemu only allows
3366                 # connecting UART devices via the '-serial' command line, as
3367                 # the device has a fixed slot on the hardware...
3368                 if ($arch eq 'aarch64' && $i == 0) {
3369                     push @$devices, '-serial', "chardev:serial$i";
3370                 } else {
3371                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3372                 }
3373             } else {
3374                 die "no such serial device\n" if ! -c $path;
3375                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3376                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3377             }
3378         }
3379     }
3380
3381     # parallel devices
3382     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3383         if (my $path = $conf->{"parallel$i"}) {
3384             die "no such parallel device\n" if ! -c $path;
3385             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3386             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3387             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3388         }
3389     }
3390
3391     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3392         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3393         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3394         push @$devices, @$audio_devs;
3395     }
3396
3397     my $sockets = 1;
3398     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3399     $sockets = $conf->{sockets} if  $conf->{sockets};
3400
3401     my $cores = $conf->{cores} || 1;
3402
3403     my $maxcpus = $sockets * $cores;
3404
3405     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3406
3407     my $allowed_vcpus = $cpuinfo->{cpus};
3408
3409     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3410         if ($allowed_vcpus < $maxcpus);
3411
3412     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3413
3414         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3415         for (my $i = 2; $i <= $vcpus; $i++)  {
3416             my $cpustr = print_cpu_device($conf,$i);
3417             push @$cmd, '-device', $cpustr;
3418         }
3419
3420     } else {
3421
3422         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3423     }
3424     push @$cmd, '-nodefaults';
3425
3426     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3427
3428     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3429
3430     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3431
3432     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3433         push @$devices, '-device', print_vga_device(
3434             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3435         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3436         push @$cmd,  '-vnc', "unix:$socket,password";
3437     } else {
3438         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3439         push @$cmd, '-nographic';
3440     }
3441
3442     # time drift fix
3443     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3444     my $useLocaltime = $conf->{localtime};
3445
3446     if ($winversion >= 5) { # windows
3447         $useLocaltime = 1 if !defined($conf->{localtime});
3448
3449         # use time drift fix when acpi is enabled
3450         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3451             $tdf = 1 if !defined($conf->{tdf});
3452         }
3453     }
3454
3455     if ($winversion >= 6) {
3456         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3457         push @$cmd, '-no-hpet';
3458     }
3459
3460     push @$rtcFlags, 'driftfix=slew' if $tdf;
3461
3462     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3463         push @$rtcFlags, "base=$conf->{startdate}";
3464     } elsif ($useLocaltime) {
3465         push @$rtcFlags, 'base=localtime';
3466     }
3467
3468     if ($forcecpu) {
3469         push @$cmd, '-cpu', $forcecpu;
3470     } else {
3471         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3472     }
3473
3474     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3475
3476     push @$cmd, '-S' if $conf->{freeze};
3477
3478     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3479
3480     my $guest_agent = parse_guest_agent($conf);
3481
3482     if ($guest_agent->{enabled}) {
3483         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3484         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3485
3486         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3487             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3488             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3489             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3490         } elsif ($guest_agent->{type} eq 'isa') {
3491             push @$devices, '-device', "isa-serial,chardev=qga0";
3492         }
3493     }
3494
3495     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3496     if ($rng && $version_guard->(4, 1, 2)) {
3497         check_rng_source($rng->{source});
3498
3499         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3500         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3501         my $limiter_str = "";
3502         if ($max_bytes) {
3503             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3504         }
3505
3506         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3507         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3508         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3509     }
3510
3511     my $spice_port;
3512
3513     if ($qxlnum) {
3514         if ($qxlnum > 1) {
3515             if ($winversion){
3516                 for (my $i = 1; $i < $qxlnum; $i++){
3517                     push @$devices, '-device', print_vga_device(
3518                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3519                 }
3520             } else {
3521                 # assume other OS works like Linux
3522                 my ($ram, $vram) = ("134217728", "67108864");
3523                 if ($vga->{memory}) {
3524                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3525                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3526                 }
3527                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3528                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3529             }
3530         }
3531
3532         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3533
3534         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3535         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3536         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3537
3538         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3539         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3540         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3541
3542         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3543         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3544
3545         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3546         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3547         if ($spice_enhancement->{foldersharing}) {
3548             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3549             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3550         }
3551
3552         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3553         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3554             if $spice_enhancement->{videostreaming};
3555
3556         push @$devices, '-spice', "$spice_opts";
3557     }
3558
3559     # enable balloon by default, unless explicitly disabled
3560     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3561         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3562         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3563     }
3564
3565     if ($conf->{watchdog}) {
3566         my $wdopts = parse_watchdog($conf->{watchdog});
3567         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3568         my $watchdog = $wdopts->{model} || 'i6300esb';
3569         push @$devices, '-device', "$watchdog$pciaddr";
3570         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3571     }
3572
3573     my $vollist = [];
3574     my $scsicontroller = {};
3575     my $ahcicontroller = {};
3576     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3577
3578     # Add iscsi initiator name if available
3579     if (my $initiator = get_initiator_name()) {
3580         push @$devices, '-iscsi', "initiator-name=$initiator";
3581     }
3582
3583     PVE::QemuConfig->foreach_volume($conf, sub {
3584         my ($ds, $drive) = @_;
3585
3586         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3587             push @$vollist, $drive->{file};
3588         }
3589
3590         # ignore efidisk here, already added in bios/fw handling code above
3591         return if $drive->{interface} eq 'efidisk';
3592
3593         $use_virtio = 1 if $ds =~ m/^virtio/;
3594
3595         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3596
3597         if ($drive->{interface} eq 'virtio'){
3598            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3599         }
3600
3601         if ($drive->{interface} eq 'scsi') {
3602
3603             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3604
3605             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3606                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3607
3608             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3609             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3610
3611             my $iothread = '';
3612             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3613                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3614                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3615             } elsif ($drive->{iothread}) {
3616                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3617             }
3618
3619             my $queues = '';
3620             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3621                 $queues = ",num_queues=$drive->{queues}";
3622             }
3623
3624             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3625                 if !$scsicontroller->{$controller};
3626             $scsicontroller->{$controller}=1;
3627         }
3628
3629         if ($drive->{interface} eq 'sata') {
3630             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3631             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3632             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3633                 if !$ahcicontroller->{$controller};
3634             $ahcicontroller->{$controller}=1;
3635         }
3636
3637         my $pbs_conf = $pbs_backing->{$ds};
3638         my $pbs_name = undef;
3639         if ($pbs_conf) {
3640             $pbs_name = "drive-$ds-pbs";
3641             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3642         }
3643
3644         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3645         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3646
3647         push @$devices, '-drive',$drive_cmd;
3648         push @$devices, '-device', print_drivedevice_full(
3649             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3650     });
3651
3652     for (my $i = 0; $i < $MAX_NETS; $i++) {
3653         my $netname = "net$i";
3654
3655         next if !$conf->{$netname};
3656         my $d = parse_net($conf->{$netname});
3657         next if !$d;
3658
3659         $use_virtio = 1 if $d->{model} eq 'virtio';
3660
3661         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3662
3663         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3664         push @$devices, '-netdev', $netdevfull;
3665
3666         my $netdevicefull = print_netdevice_full(
3667             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3668
3669         push @$devices, '-device', $netdevicefull;
3670     }
3671
3672     if ($conf->{ivshmem}) {
3673         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3674
3675         my $bus;
3676         if ($q35) {
3677             $bus = print_pcie_addr("ivshmem");
3678         } else {
3679             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3680         }
3681
3682         my $ivshmem_name = $ivshmem->{name} // $vmid;
3683         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3684
3685         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3686         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3687             .",size=$ivshmem->{size}M";
3688     }
3689
3690     # pci.4 is nested in pci.1
3691     $bridges->{1} = 1 if $bridges->{4};
3692
3693     if (!$q35) {
3694         # add pci bridges
3695         if (min_version($machine_version, 2, 3)) {
3696            $bridges->{1} = 1;
3697            $bridges->{2} = 1;
3698         }
3699
3700         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3701
3702     }
3703
3704     for my $k (sort {$b cmp $a} keys %$bridges) {
3705         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3706
3707         my $k_name = $k;
3708         if ($k == 2 && $legacy_igd) {
3709             $k_name = "$k-igd";
3710         }
3711         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3712
3713         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3714         if ($q35) {
3715             # add after -readconfig pve-q35.cfg
3716             splice @$devices, 2, 0, '-device', $devstr;
3717         } else {
3718             unshift @$devices, '-device', $devstr if $k > 0;
3719         }
3720     }
3721
3722     if (!$kvm) {
3723         push @$machineFlags, 'accel=tcg';
3724     }
3725
3726     my $machine_type_min = $machine_type;
3727     if ($add_pve_version) {
3728         $machine_type_min =~ s/\+pve\d+$//;
3729         $machine_type_min .= "+pve$required_pve_version";
3730     }
3731     push @$machineFlags, "type=${machine_type_min}";
3732
3733     push @$cmd, @$devices;
3734     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3735     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3736     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3737
3738     if (my $vmstate = $conf->{vmstate}) {
3739         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3740         push @$vollist, $vmstate;
3741         push @$cmd, '-loadstate', $statepath;
3742         print "activating and using '$vmstate' as vmstate\n";
3743     }
3744
3745     # add custom args
3746     if ($conf->{args}) {
3747         my $aa = PVE::Tools::split_args($conf->{args});
3748         push @$cmd, @$aa;
3749     }
3750
3751     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3752 }
3753
3754 sub check_rng_source {
3755     my ($source) = @_;
3756
3757     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3758     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3759         if ! -e $source;
3760
3761     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3762     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3763         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3764         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3765         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3766             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3767             ." to the host.\n";
3768     }
3769 }
3770
3771 sub spice_port {
3772     my ($vmid) = @_;
3773
3774     my $res = mon_cmd($vmid, 'query-spice');
3775
3776     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3777 }
3778
3779 sub vm_devices_list {
3780     my ($vmid) = @_;
3781
3782     my $res = mon_cmd($vmid, 'query-pci');
3783     my $devices_to_check = [];
3784     my $devices = {};
3785     foreach my $pcibus (@$res) {
3786         push @$devices_to_check, @{$pcibus->{devices}},
3787     }
3788
3789     while (@$devices_to_check) {
3790         my $to_check = [];
3791         for my $d (@$devices_to_check) {
3792             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3793             next if !$d->{'pci_bridge'};
3794
3795             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3796             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3797         }
3798         $devices_to_check = $to_check;
3799     }
3800
3801     my $resblock = mon_cmd($vmid, 'query-block');
3802     foreach my $block (@$resblock) {
3803         if($block->{device} =~ m/^drive-(\S+)/){
3804                 $devices->{$1} = 1;
3805         }
3806     }
3807
3808     my $resmice = mon_cmd($vmid, 'query-mice');
3809     foreach my $mice (@$resmice) {
3810         if ($mice->{name} eq 'QEMU HID Tablet') {
3811             $devices->{tablet} = 1;
3812             last;
3813         }
3814     }
3815
3816     # for usb devices there is no query-usb
3817     # but we can iterate over the entries in
3818     # qom-list path=/machine/peripheral
3819     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3820     foreach my $per (@$resperipheral) {
3821         if ($per->{name} =~ m/^usb\d+$/) {
3822             $devices->{$per->{name}} = 1;
3823         }
3824     }
3825
3826     return $devices;
3827 }
3828
3829 sub vm_deviceplug {
3830     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3831
3832     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3833
3834     my $devices_list = vm_devices_list($vmid);
3835     return 1 if defined($devices_list->{$deviceid});
3836
3837     # add PCI bridge if we need it for the device
3838     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3839
3840     if ($deviceid eq 'tablet') {
3841
3842         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3843
3844     } elsif ($deviceid eq 'keyboard') {
3845
3846         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3847
3848     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3849
3850         die "usb hotplug currently not reliable\n";
3851         # since we can't reliably hot unplug all added usb devices and usb
3852         # passthrough breaks live migration we disable usb hotplugging for now
3853         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3854
3855     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3856
3857         qemu_iothread_add($vmid, $deviceid, $device);
3858
3859         qemu_driveadd($storecfg, $vmid, $device);
3860         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3861
3862         qemu_deviceadd($vmid, $devicefull);
3863         eval { qemu_deviceaddverify($vmid, $deviceid); };
3864         if (my $err = $@) {
3865             eval { qemu_drivedel($vmid, $deviceid); };
3866             warn $@ if $@;
3867             die $err;
3868         }
3869
3870     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3871
3872
3873         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3874         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3875         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3876
3877         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3878
3879         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3880             qemu_iothread_add($vmid, $deviceid, $device);
3881             $devicefull .= ",iothread=iothread-$deviceid";
3882         }
3883
3884         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3885             $devicefull .= ",num_queues=$device->{queues}";
3886         }
3887
3888         qemu_deviceadd($vmid, $devicefull);
3889         qemu_deviceaddverify($vmid, $deviceid);
3890
3891     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3892
3893         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3894         qemu_driveadd($storecfg, $vmid, $device);
3895
3896         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3897         eval { qemu_deviceadd($vmid, $devicefull); };
3898         if (my $err = $@) {
3899             eval { qemu_drivedel($vmid, $deviceid); };
3900             warn $@ if $@;
3901             die $err;
3902         }
3903
3904     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3905
3906         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3907
3908         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3909         my $use_old_bios_files = undef;
3910         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3911
3912         my $netdevicefull = print_netdevice_full(
3913             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3914         qemu_deviceadd($vmid, $netdevicefull);
3915         eval {
3916             qemu_deviceaddverify($vmid, $deviceid);
3917             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3918         };
3919         if (my $err = $@) {
3920             eval { qemu_netdevdel($vmid, $deviceid); };
3921             warn $@ if $@;
3922             die $err;
3923         }
3924
3925     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3926
3927         my $bridgeid = $2;
3928         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3929         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3930
3931         qemu_deviceadd($vmid, $devicefull);
3932         qemu_deviceaddverify($vmid, $deviceid);
3933
3934     } else {
3935         die "can't hotplug device '$deviceid'\n";
3936     }
3937
3938     return 1;
3939 }
3940
3941 # fixme: this should raise exceptions on error!
3942 sub vm_deviceunplug {
3943     my ($vmid, $conf, $deviceid) = @_;
3944
3945     my $devices_list = vm_devices_list($vmid);
3946     return 1 if !defined($devices_list->{$deviceid});
3947
3948     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3949     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3950
3951     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3952
3953         qemu_devicedel($vmid, $deviceid);
3954
3955     } elsif ($deviceid =~ m/^usb\d+$/) {
3956
3957         die "usb hotplug currently not reliable\n";
3958         # when unplugging usb devices this way, there may be remaining usb
3959         # controllers/hubs so we disable it for now
3960         #qemu_devicedel($vmid, $deviceid);
3961         #qemu_devicedelverify($vmid, $deviceid);
3962
3963     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3964
3965         qemu_devicedel($vmid, $deviceid);
3966         qemu_devicedelverify($vmid, $deviceid);
3967         qemu_drivedel($vmid, $deviceid);
3968         qemu_iothread_del($conf, $vmid, $deviceid);
3969
3970     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3971
3972         qemu_devicedel($vmid, $deviceid);
3973         qemu_devicedelverify($vmid, $deviceid);
3974         qemu_iothread_del($conf, $vmid, $deviceid);
3975
3976     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3977
3978         qemu_devicedel($vmid, $deviceid);
3979         qemu_drivedel($vmid, $deviceid);
3980         qemu_deletescsihw($conf, $vmid, $deviceid);
3981
3982     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3983
3984         qemu_devicedel($vmid, $deviceid);
3985         qemu_devicedelverify($vmid, $deviceid);
3986         qemu_netdevdel($vmid, $deviceid);
3987
3988     } else {
3989         die "can't unplug device '$deviceid'\n";
3990     }
3991
3992     return 1;
3993 }
3994
3995 sub qemu_deviceadd {
3996     my ($vmid, $devicefull) = @_;
3997
3998     $devicefull = "driver=".$devicefull;
3999     my %options =  split(/[=,]/, $devicefull);
4000
4001     mon_cmd($vmid, "device_add" , %options);
4002 }
4003
4004 sub qemu_devicedel {
4005     my ($vmid, $deviceid) = @_;
4006
4007     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4008 }
4009
4010 sub qemu_iothread_add {
4011     my($vmid, $deviceid, $device) = @_;
4012
4013     if ($device->{iothread}) {
4014         my $iothreads = vm_iothreads_list($vmid);
4015         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4016     }
4017 }
4018
4019 sub qemu_iothread_del {
4020     my($conf, $vmid, $deviceid) = @_;
4021
4022     my $confid = $deviceid;
4023     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4024         $confid = 'scsi' . $1;
4025     }
4026     my $device = parse_drive($confid, $conf->{$confid});
4027     if ($device->{iothread}) {
4028         my $iothreads = vm_iothreads_list($vmid);
4029         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4030     }
4031 }
4032
4033 sub qemu_objectadd {
4034     my($vmid, $objectid, $qomtype) = @_;
4035
4036     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4037
4038     return 1;
4039 }
4040
4041 sub qemu_objectdel {
4042     my($vmid, $objectid) = @_;
4043
4044     mon_cmd($vmid, "object-del", id => $objectid);
4045
4046     return 1;
4047 }
4048
4049 sub qemu_driveadd {
4050     my ($storecfg, $vmid, $device) = @_;
4051
4052     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4053     $drive =~ s/\\/\\\\/g;
4054     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4055
4056     # If the command succeeds qemu prints: "OK"
4057     return 1 if $ret =~ m/OK/s;
4058
4059     die "adding drive failed: $ret\n";
4060 }
4061
4062 sub qemu_drivedel {
4063     my($vmid, $deviceid) = @_;
4064
4065     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4066     $ret =~ s/^\s+//;
4067
4068     return 1 if $ret eq "";
4069
4070     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4071     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4072
4073     die "deleting drive $deviceid failed : $ret\n";
4074 }
4075
4076 sub qemu_deviceaddverify {
4077     my ($vmid, $deviceid) = @_;
4078
4079     for (my $i = 0; $i <= 5; $i++) {
4080          my $devices_list = vm_devices_list($vmid);
4081          return 1 if defined($devices_list->{$deviceid});
4082          sleep 1;
4083     }
4084
4085     die "error on hotplug device '$deviceid'\n";
4086 }
4087
4088
4089 sub qemu_devicedelverify {
4090     my ($vmid, $deviceid) = @_;
4091
4092     # need to verify that the device is correctly removed as device_del
4093     # is async and empty return is not reliable
4094
4095     for (my $i = 0; $i <= 5; $i++) {
4096          my $devices_list = vm_devices_list($vmid);
4097          return 1 if !defined($devices_list->{$deviceid});
4098          sleep 1;
4099     }
4100
4101     die "error on hot-unplugging device '$deviceid'\n";
4102 }
4103
4104 sub qemu_findorcreatescsihw {
4105     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4106
4107     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4108
4109     my $scsihwid="$controller_prefix$controller";
4110     my $devices_list = vm_devices_list($vmid);
4111
4112     if(!defined($devices_list->{$scsihwid})) {
4113         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4114     }
4115
4116     return 1;
4117 }
4118
4119 sub qemu_deletescsihw {
4120     my ($conf, $vmid, $opt) = @_;
4121
4122     my $device = parse_drive($opt, $conf->{$opt});
4123
4124     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4125         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4126         return 1;
4127     }
4128
4129     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4130
4131     my $devices_list = vm_devices_list($vmid);
4132     foreach my $opt (keys %{$devices_list}) {
4133         if (is_valid_drivename($opt)) {
4134             my $drive = parse_drive($opt, $conf->{$opt});
4135             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4136                 return 1;
4137             }
4138         }
4139     }
4140
4141     my $scsihwid="scsihw$controller";
4142
4143     vm_deviceunplug($vmid, $conf, $scsihwid);
4144
4145     return 1;
4146 }
4147
4148 sub qemu_add_pci_bridge {
4149     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4150
4151     my $bridges = {};
4152
4153     my $bridgeid;
4154
4155     print_pci_addr($device, $bridges, $arch, $machine_type);
4156
4157     while (my ($k, $v) = each %$bridges) {
4158         $bridgeid = $k;
4159     }
4160     return 1 if !defined($bridgeid) || $bridgeid < 1;
4161
4162     my $bridge = "pci.$bridgeid";
4163     my $devices_list = vm_devices_list($vmid);
4164
4165     if (!defined($devices_list->{$bridge})) {
4166         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4167     }
4168
4169     return 1;
4170 }
4171
4172 sub qemu_set_link_status {
4173     my ($vmid, $device, $up) = @_;
4174
4175     mon_cmd($vmid, "set_link", name => $device,
4176                up => $up ? JSON::true : JSON::false);
4177 }
4178
4179 sub qemu_netdevadd {
4180     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4181
4182     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4183     my %options =  split(/[=,]/, $netdev);
4184
4185     if (defined(my $vhost = $options{vhost})) {
4186         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4187     }
4188
4189     if (defined(my $queues = $options{queues})) {
4190         $options{queues} = $queues + 0;
4191     }
4192
4193     mon_cmd($vmid, "netdev_add",  %options);
4194     return 1;
4195 }
4196
4197 sub qemu_netdevdel {
4198     my ($vmid, $deviceid) = @_;
4199
4200     mon_cmd($vmid, "netdev_del", id => $deviceid);
4201 }
4202
4203 sub qemu_usb_hotplug {
4204     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4205
4206     return if !$device;
4207
4208     # remove the old one first
4209     vm_deviceunplug($vmid, $conf, $deviceid);
4210
4211     # check if xhci controller is necessary and available
4212     if ($device->{usb3}) {
4213
4214         my $devicelist = vm_devices_list($vmid);
4215
4216         if (!$devicelist->{xhci}) {
4217             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4218             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4219         }
4220     }
4221     my $d = parse_usb_device($device->{host});
4222     $d->{usb3} = $device->{usb3};
4223
4224     # add the new one
4225     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4226 }
4227
4228 sub qemu_cpu_hotplug {
4229     my ($vmid, $conf, $vcpus) = @_;
4230
4231     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4232
4233     my $sockets = 1;
4234     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4235     $sockets = $conf->{sockets} if  $conf->{sockets};
4236     my $cores = $conf->{cores} || 1;
4237     my $maxcpus = $sockets * $cores;
4238
4239     $vcpus = $maxcpus if !$vcpus;
4240
4241     die "you can't add more vcpus than maxcpus\n"
4242         if $vcpus > $maxcpus;
4243
4244     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4245
4246     if ($vcpus < $currentvcpus) {
4247
4248         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4249
4250             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4251                 qemu_devicedel($vmid, "cpu$i");
4252                 my $retry = 0;
4253                 my $currentrunningvcpus = undef;
4254                 while (1) {
4255                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4256                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4257                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4258                     $retry++;
4259                     sleep 1;
4260                 }
4261                 #update conf after each succesfull cpu unplug
4262                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4263                 PVE::QemuConfig->write_config($vmid, $conf);
4264             }
4265         } else {
4266             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4267         }
4268
4269         return;
4270     }
4271
4272     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4273     die "vcpus in running vm does not match its configuration\n"
4274         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4275
4276     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4277
4278         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4279             my $cpustr = print_cpu_device($conf, $i);
4280             qemu_deviceadd($vmid, $cpustr);
4281
4282             my $retry = 0;
4283             my $currentrunningvcpus = undef;
4284             while (1) {
4285                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4286                 last if scalar(@{$currentrunningvcpus}) == $i;
4287                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4288                 sleep 1;
4289                 $retry++;
4290             }
4291             #update conf after each succesfull cpu hotplug
4292             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4293             PVE::QemuConfig->write_config($vmid, $conf);
4294         }
4295     } else {
4296
4297         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4298             mon_cmd($vmid, "cpu-add", id => int($i));
4299         }
4300     }
4301 }
4302
4303 sub qemu_block_set_io_throttle {
4304     my ($vmid, $deviceid,
4305         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4306         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4307         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4308         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4309
4310     return if !check_running($vmid) ;
4311
4312     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4313         bps => int($bps),
4314         bps_rd => int($bps_rd),
4315         bps_wr => int($bps_wr),
4316         iops => int($iops),
4317         iops_rd => int($iops_rd),
4318         iops_wr => int($iops_wr),
4319         bps_max => int($bps_max),
4320         bps_rd_max => int($bps_rd_max),
4321         bps_wr_max => int($bps_wr_max),
4322         iops_max => int($iops_max),
4323         iops_rd_max => int($iops_rd_max),
4324         iops_wr_max => int($iops_wr_max),
4325         bps_max_length => int($bps_max_length),
4326         bps_rd_max_length => int($bps_rd_max_length),
4327         bps_wr_max_length => int($bps_wr_max_length),
4328         iops_max_length => int($iops_max_length),
4329         iops_rd_max_length => int($iops_rd_max_length),
4330         iops_wr_max_length => int($iops_wr_max_length),
4331     );
4332
4333 }
4334
4335 sub qemu_block_resize {
4336     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4337
4338     my $running = check_running($vmid);
4339
4340     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4341
4342     return if !$running;
4343
4344     my $padding = (1024 - $size % 1024) % 1024;
4345     $size = $size + $padding;
4346
4347     mon_cmd(
4348         $vmid,
4349         "block_resize",
4350         device => $deviceid,
4351         size => int($size),
4352         timeout => 60,
4353     );
4354 }
4355
4356 sub qemu_volume_snapshot {
4357     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4358
4359     my $running = check_running($vmid);
4360
4361     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4362         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4363     } else {
4364         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4365     }
4366 }
4367
4368 sub qemu_volume_snapshot_delete {
4369     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4370
4371     my $running = check_running($vmid);
4372
4373     if($running) {
4374
4375         $running = undef;
4376         my $conf = PVE::QemuConfig->load_config($vmid);
4377         PVE::QemuConfig->foreach_volume($conf, sub {
4378             my ($ds, $drive) = @_;
4379             $running = 1 if $drive->{file} eq $volid;
4380         });
4381     }
4382
4383     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4384         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4385     } else {
4386         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4387     }
4388 }
4389
4390 sub set_migration_caps {
4391     my ($vmid, $savevm) = @_;
4392
4393     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4394
4395     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4396     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4397
4398     my $cap_ref = [];
4399
4400     my $enabled_cap = {
4401         "auto-converge" => 1,
4402         "xbzrle" => 1,
4403         "x-rdma-pin-all" => 0,
4404         "zero-blocks" => 0,
4405         "compress" => 0,
4406         "dirty-bitmaps" => $dirty_bitmaps,
4407     };
4408
4409     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4410
4411     for my $supported_capability (@$supported_capabilities) {
4412         push @$cap_ref, {
4413             capability => $supported_capability->{capability},
4414             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4415         };
4416     }
4417
4418     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4419 }
4420
4421 sub foreach_volid {
4422     my ($conf, $func, @param) = @_;
4423
4424     my $volhash = {};
4425
4426     my $test_volid = sub {
4427         my ($key, $drive, $snapname) = @_;
4428
4429         my $volid = $drive->{file};
4430         return if !$volid;
4431
4432         $volhash->{$volid}->{cdrom} //= 1;
4433         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4434
4435         my $replicate = $drive->{replicate} // 1;
4436         $volhash->{$volid}->{replicate} //= 0;
4437         $volhash->{$volid}->{replicate} = 1 if $replicate;
4438
4439         $volhash->{$volid}->{shared} //= 0;
4440         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4441
4442         $volhash->{$volid}->{referenced_in_config} //= 0;
4443         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4444
4445         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4446             if defined($snapname);
4447
4448         my $size = $drive->{size};
4449         $volhash->{$volid}->{size} //= $size if $size;
4450
4451         $volhash->{$volid}->{is_vmstate} //= 0;
4452         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4453
4454         $volhash->{$volid}->{is_unused} //= 0;
4455         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4456     };
4457
4458     my $include_opts = {
4459         extra_keys => ['vmstate'],
4460         include_unused => 1,
4461     };
4462
4463     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4464     foreach my $snapname (keys %{$conf->{snapshots}}) {
4465         my $snap = $conf->{snapshots}->{$snapname};
4466         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4467     }
4468
4469     foreach my $volid (keys %$volhash) {
4470         &$func($volid, $volhash->{$volid}, @param);
4471     }
4472 }
4473
4474 my $fast_plug_option = {
4475     'lock' => 1,
4476     'name' => 1,
4477     'onboot' => 1,
4478     'shares' => 1,
4479     'startup' => 1,
4480     'description' => 1,
4481     'protection' => 1,
4482     'vmstatestorage' => 1,
4483     'hookscript' => 1,
4484     'tags' => 1,
4485 };
4486
4487 # hotplug changes in [PENDING]
4488 # $selection hash can be used to only apply specified options, for
4489 # example: { cores => 1 } (only apply changed 'cores')
4490 # $errors ref is used to return error messages
4491 sub vmconfig_hotplug_pending {
4492     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4493
4494     my $defaults = load_defaults();
4495     my $arch = get_vm_arch($conf);
4496     my $machine_type = get_vm_machine($conf, undef, $arch);
4497
4498     # commit values which do not have any impact on running VM first
4499     # Note: those option cannot raise errors, we we do not care about
4500     # $selection and always apply them.
4501
4502     my $add_error = sub {
4503         my ($opt, $msg) = @_;
4504         $errors->{$opt} = "hotplug problem - $msg";
4505     };
4506
4507     my $changes = 0;
4508     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4509         if ($fast_plug_option->{$opt}) {
4510             $conf->{$opt} = $conf->{pending}->{$opt};
4511             delete $conf->{pending}->{$opt};
4512             $changes = 1;
4513         }
4514     }
4515
4516     if ($changes) {
4517         PVE::QemuConfig->write_config($vmid, $conf);
4518     }
4519
4520     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4521
4522     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4523     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4524     foreach my $opt (sort keys %$pending_delete_hash) {
4525         next if $selection && !$selection->{$opt};
4526         my $force = $pending_delete_hash->{$opt}->{force};
4527         eval {
4528             if ($opt eq 'hotplug') {
4529                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4530             } elsif ($opt eq 'tablet') {
4531                 die "skip\n" if !$hotplug_features->{usb};
4532                 if ($defaults->{tablet}) {
4533                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4534                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4535                         if $arch eq 'aarch64';
4536                 } else {
4537                     vm_deviceunplug($vmid, $conf, 'tablet');
4538                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4539                 }
4540             } elsif ($opt =~ m/^usb\d+/) {
4541                 die "skip\n";
4542                 # since we cannot reliably hot unplug usb devices we are disabling it
4543                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4544                 #vm_deviceunplug($vmid, $conf, $opt);
4545             } elsif ($opt eq 'vcpus') {
4546                 die "skip\n" if !$hotplug_features->{cpu};
4547                 qemu_cpu_hotplug($vmid, $conf, undef);
4548             } elsif ($opt eq 'balloon') {
4549                 # enable balloon device is not hotpluggable
4550                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4551                 # here we reset the ballooning value to memory
4552                 my $balloon = $conf->{memory} || $defaults->{memory};
4553                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4554             } elsif ($fast_plug_option->{$opt}) {
4555                 # do nothing
4556             } elsif ($opt =~ m/^net(\d+)$/) {
4557                 die "skip\n" if !$hotplug_features->{network};
4558                 vm_deviceunplug($vmid, $conf, $opt);
4559             } elsif (is_valid_drivename($opt)) {
4560                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4561                 vm_deviceunplug($vmid, $conf, $opt);
4562                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4563             } elsif ($opt =~ m/^memory$/) {
4564                 die "skip\n" if !$hotplug_features->{memory};
4565                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4566             } elsif ($opt eq 'cpuunits') {
4567                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4568             } elsif ($opt eq 'cpulimit') {
4569                 $cgroup->change_cpu_quota(-1, 100000);
4570             } else {
4571                 die "skip\n";
4572             }
4573         };
4574         if (my $err = $@) {
4575             &$add_error($opt, $err) if $err ne "skip\n";
4576         } else {
4577             delete $conf->{$opt};
4578             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4579         }
4580     }
4581
4582     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4583     $apply_pending_cloudinit = sub {
4584         return if $apply_pending_cloudinit_done; # once is enough
4585         $apply_pending_cloudinit_done = 1; # once is enough
4586
4587         my ($key, $value) = @_;
4588
4589         my @cloudinit_opts = keys %$confdesc_cloudinit;
4590         foreach my $opt (keys %{$conf->{pending}}) {
4591             next if !grep { $_ eq $opt } @cloudinit_opts;
4592             $conf->{$opt} = delete $conf->{pending}->{$opt};
4593         }
4594
4595         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4596         foreach my $opt (sort keys %$pending_delete_hash) {
4597             next if !grep { $_ eq $opt } @cloudinit_opts;
4598             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4599             delete $conf->{$opt};
4600         }
4601
4602         my $new_conf = { %$conf };
4603         $new_conf->{$key} = $value;
4604         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4605     };
4606
4607     foreach my $opt (keys %{$conf->{pending}}) {
4608         next if $selection && !$selection->{$opt};
4609         my $value = $conf->{pending}->{$opt};
4610         eval {
4611             if ($opt eq 'hotplug') {
4612                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4613             } elsif ($opt eq 'tablet') {
4614                 die "skip\n" if !$hotplug_features->{usb};
4615                 if ($value == 1) {
4616                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4617                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4618                         if $arch eq 'aarch64';
4619                 } elsif ($value == 0) {
4620                     vm_deviceunplug($vmid, $conf, 'tablet');
4621                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4622                 }
4623             } elsif ($opt =~ m/^usb\d+$/) {
4624                 die "skip\n";
4625                 # since we cannot reliably hot unplug usb devices we disable it for now
4626                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4627                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4628                 #die "skip\n" if !$d;
4629                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4630             } elsif ($opt eq 'vcpus') {
4631                 die "skip\n" if !$hotplug_features->{cpu};
4632                 qemu_cpu_hotplug($vmid, $conf, $value);
4633             } elsif ($opt eq 'balloon') {
4634                 # enable/disable balloning device is not hotpluggable
4635                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4636                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4637                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4638
4639                 # allow manual ballooning if shares is set to zero
4640                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4641                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4642                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4643                 }
4644             } elsif ($opt =~ m/^net(\d+)$/) {
4645                 # some changes can be done without hotplug
4646                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4647                                     $vmid, $opt, $value, $arch, $machine_type);
4648             } elsif (is_valid_drivename($opt)) {
4649                 die "skip\n" if $opt eq 'efidisk0';
4650                 # some changes can be done without hotplug
4651                 my $drive = parse_drive($opt, $value);
4652                 if (drive_is_cloudinit($drive)) {
4653                     &$apply_pending_cloudinit($opt, $value);
4654                 }
4655                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4656                                      $vmid, $opt, $value, $arch, $machine_type);
4657             } elsif ($opt =~ m/^memory$/) { #dimms
4658                 die "skip\n" if !$hotplug_features->{memory};
4659                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4660             } elsif ($opt eq 'cpuunits') {
4661                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4662             } elsif ($opt eq 'cpulimit') {
4663                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4664                 $cgroup->change_cpu_quota($cpulimit, 100000);
4665             } else {
4666                 die "skip\n";  # skip non-hot-pluggable options
4667             }
4668         };
4669         if (my $err = $@) {
4670             &$add_error($opt, $err) if $err ne "skip\n";
4671         } else {
4672             $conf->{$opt} = $value;
4673             delete $conf->{pending}->{$opt};
4674         }
4675     }
4676
4677     PVE::QemuConfig->write_config($vmid, $conf);
4678 }
4679
4680 sub try_deallocate_drive {
4681     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4682
4683     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4684         my $volid = $drive->{file};
4685         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4686             my $sid = PVE::Storage::parse_volume_id($volid);
4687             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4688
4689             # check if the disk is really unused
4690             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4691                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4692             PVE::Storage::vdisk_free($storecfg, $volid);
4693             return 1;
4694         } else {
4695             # If vm is not owner of this disk remove from config
4696             return 1;
4697         }
4698     }
4699
4700     return;
4701 }
4702
4703 sub vmconfig_delete_or_detach_drive {
4704     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4705
4706     my $drive = parse_drive($opt, $conf->{$opt});
4707
4708     my $rpcenv = PVE::RPCEnvironment::get();
4709     my $authuser = $rpcenv->get_user();
4710
4711     if ($force) {
4712         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4713         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4714     } else {
4715         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4716     }
4717 }
4718
4719
4720
4721 sub vmconfig_apply_pending {
4722     my ($vmid, $conf, $storecfg, $errors) = @_;
4723
4724     my $add_apply_error = sub {
4725         my ($opt, $msg) = @_;
4726         my $err_msg = "unable to apply pending change $opt : $msg";
4727         $errors->{$opt} = $err_msg;
4728         warn $err_msg;
4729     };
4730
4731     # cold plug
4732
4733     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4734     foreach my $opt (sort keys %$pending_delete_hash) {
4735         my $force = $pending_delete_hash->{$opt}->{force};
4736         eval {
4737             if ($opt =~ m/^unused/) {
4738                 die "internal error";
4739             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4740                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4741             }
4742         };
4743         if (my $err = $@) {
4744             $add_apply_error->($opt, $err);
4745         } else {
4746             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4747             delete $conf->{$opt};
4748         }
4749     }
4750
4751     PVE::QemuConfig->cleanup_pending($conf);
4752
4753     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4754         next if $opt eq 'delete'; # just to be sure
4755         eval {
4756             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4757                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4758             }
4759         };
4760         if (my $err = $@) {
4761             $add_apply_error->($opt, $err);
4762         } else {
4763             $conf->{$opt} = delete $conf->{pending}->{$opt};
4764         }
4765     }
4766
4767     # write all changes at once to avoid unnecessary i/o
4768     PVE::QemuConfig->write_config($vmid, $conf);
4769 }
4770
4771 sub vmconfig_update_net {
4772     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4773
4774     my $newnet = parse_net($value);
4775
4776     if ($conf->{$opt}) {
4777         my $oldnet = parse_net($conf->{$opt});
4778
4779         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4780             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4781             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4782             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4783
4784             # for non online change, we try to hot-unplug
4785             die "skip\n" if !$hotplug;
4786             vm_deviceunplug($vmid, $conf, $opt);
4787         } else {
4788
4789             die "internal error" if $opt !~ m/net(\d+)/;
4790             my $iface = "tap${vmid}i$1";
4791
4792             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4793                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4794                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4795                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4796                 PVE::Network::tap_unplug($iface);
4797
4798                 if ($have_sdn) {
4799                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4800                 } else {
4801                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4802                 }
4803             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4804                 # Rate can be applied on its own but any change above needs to
4805                 # include the rate in tap_plug since OVS resets everything.
4806                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4807             }
4808
4809             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4810                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4811             }
4812
4813             return 1;
4814         }
4815     }
4816
4817     if ($hotplug) {
4818         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4819     } else {
4820         die "skip\n";
4821     }
4822 }
4823
4824 sub vmconfig_update_disk {
4825     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4826
4827     my $drive = parse_drive($opt, $value);
4828
4829     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4830         my $media = $drive->{media} || 'disk';
4831         my $oldmedia = $old_drive->{media} || 'disk';
4832         die "unable to change media type\n" if $media ne $oldmedia;
4833
4834         if (!drive_is_cdrom($old_drive)) {
4835
4836             if ($drive->{file} ne $old_drive->{file}) {
4837
4838                 die "skip\n" if !$hotplug;
4839
4840                 # unplug and register as unused
4841                 vm_deviceunplug($vmid, $conf, $opt);
4842                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4843
4844             } else {
4845                 # update existing disk
4846
4847                 # skip non hotpluggable value
4848                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4849                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4850                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4851                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4852                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4853                     die "skip\n";
4854                 }
4855
4856                 # apply throttle
4857                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4858                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4859                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4860                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4861                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4862                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4863                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4864                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4865                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4866                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4867                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4868                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4869                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4870                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4871                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4872                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4873                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4874                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4875
4876                     qemu_block_set_io_throttle(
4877                         $vmid,"drive-$opt",
4878                         ($drive->{mbps} || 0)*1024*1024,
4879                         ($drive->{mbps_rd} || 0)*1024*1024,
4880                         ($drive->{mbps_wr} || 0)*1024*1024,
4881                          $drive->{iops} || 0,
4882                          $drive->{iops_rd} || 0,
4883                          $drive->{iops_wr} || 0,
4884                         ($drive->{mbps_max} || 0)*1024*1024,
4885                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4886                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4887                          $drive->{iops_max} || 0,
4888                          $drive->{iops_rd_max} || 0,
4889                          $drive->{iops_wr_max} || 0,
4890                          $drive->{bps_max_length} || 1,
4891                          $drive->{bps_rd_max_length} || 1,
4892                          $drive->{bps_wr_max_length} || 1,
4893                          $drive->{iops_max_length} || 1,
4894                          $drive->{iops_rd_max_length} || 1,
4895                          $drive->{iops_wr_max_length} || 1,
4896                     );
4897
4898                 }
4899
4900                 return 1;
4901             }
4902
4903         } else { # cdrom
4904
4905             if ($drive->{file} eq 'none') {
4906                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4907                 if (drive_is_cloudinit($old_drive)) {
4908                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4909                 }
4910             } else {
4911                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4912
4913                 # force eject if locked
4914                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4915
4916                 if ($path) {
4917                     mon_cmd($vmid, "blockdev-change-medium",
4918                         id => "$opt", filename => "$path");
4919                 }
4920             }
4921
4922             return 1;
4923         }
4924     }
4925
4926     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4927     # hotplug new disks
4928     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4929     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4930 }
4931
4932 # called in locked context by incoming migration
4933 sub vm_migrate_get_nbd_disks {
4934     my ($storecfg, $conf, $replicated_volumes) = @_;
4935
4936     my $local_volumes = {};
4937     PVE::QemuConfig->foreach_volume($conf, sub {
4938         my ($ds, $drive) = @_;
4939
4940         return if drive_is_cdrom($drive);
4941
4942         my $volid = $drive->{file};
4943
4944         return if !$volid;
4945
4946         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4947
4948         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4949         return if $scfg->{shared};
4950
4951         # replicated disks re-use existing state via bitmap
4952         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4953         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4954     });
4955     return $local_volumes;
4956 }
4957
4958 # called in locked context by incoming migration
4959 sub vm_migrate_alloc_nbd_disks {
4960     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4961
4962     my $format = undef;
4963
4964     my $nbd = {};
4965     foreach my $opt (sort keys %$source_volumes) {
4966         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4967
4968         if ($use_existing) {
4969             $nbd->{$opt}->{drivestr} = print_drive($drive);
4970             $nbd->{$opt}->{volid} = $volid;
4971             $nbd->{$opt}->{replicated} = 1;
4972             next;
4973         }
4974
4975         # If a remote storage is specified and the format of the original
4976         # volume is not available there, fall back to the default format.
4977         # Otherwise use the same format as the original.
4978         if (!$storagemap->{identity}) {
4979             $storeid = map_storage($storagemap, $storeid);
4980             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4981             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4982             my $fileFormat = qemu_img_format($scfg, $volname);
4983             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4984         } else {
4985             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4986             $format = qemu_img_format($scfg, $volname);
4987         }
4988
4989         my $size = $drive->{size} / 1024;
4990         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4991         my $newdrive = $drive;
4992         $newdrive->{format} = $format;
4993         $newdrive->{file} = $newvolid;
4994         my $drivestr = print_drive($newdrive);
4995         $nbd->{$opt}->{drivestr} = $drivestr;
4996         $nbd->{$opt}->{volid} = $newvolid;
4997     }
4998
4999     return $nbd;
5000 }
5001
5002 # see vm_start_nolock for parameters, additionally:
5003 # migrate_opts:
5004 #   storagemap = parsed storage map for allocating NBD disks
5005 sub vm_start {
5006     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5007
5008     return PVE::QemuConfig->lock_config($vmid, sub {
5009         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5010
5011         die "you can't start a vm if it's a template\n"
5012             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5013
5014         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5015         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5016
5017         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5018
5019         if ($has_backup_lock && $running) {
5020             # a backup is currently running, attempt to start the guest in the
5021             # existing QEMU instance
5022             return vm_resume($vmid);
5023         }
5024
5025         PVE::QemuConfig->check_lock($conf)
5026             if !($params->{skiplock} || $has_suspended_lock);
5027
5028         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5029
5030         die "VM $vmid already running\n" if $running;
5031
5032         if (my $storagemap = $migrate_opts->{storagemap}) {
5033             my $replicated = $migrate_opts->{replicated_volumes};
5034             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5035             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5036
5037             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5038                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5039             }
5040         }
5041
5042         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5043     });
5044 }
5045
5046
5047 # params:
5048 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5049 #   skiplock => 0/1, skip checking for config lock
5050 #   skiptemplate => 0/1, skip checking whether VM is template
5051 #   forcemachine => to force Qemu machine (rollback/migration)
5052 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5053 #   timeout => in seconds
5054 #   paused => start VM in paused state (backup)
5055 #   resume => resume from hibernation
5056 #   pbs-backing => {
5057 #      sata0 => {
5058 #         repository
5059 #         snapshot
5060 #         keyfile
5061 #         archive
5062 #      },
5063 #      virtio2 => ...
5064 #   }
5065 # migrate_opts:
5066 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5067 #   migratedfrom => source node
5068 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5069 #   network => CIDR of migration network
5070 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5071 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5072 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5073 sub vm_start_nolock {
5074     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5075
5076     my $statefile = $params->{statefile};
5077     my $resume = $params->{resume};
5078
5079     my $migratedfrom = $migrate_opts->{migratedfrom};
5080     my $migration_type = $migrate_opts->{type};
5081
5082     my $res = {};
5083
5084     # clean up leftover reboot request files
5085     eval { clear_reboot_request($vmid); };
5086     warn $@ if $@;
5087
5088     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5089         vmconfig_apply_pending($vmid, $conf, $storecfg);
5090         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5091     }
5092
5093     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5094
5095     my $defaults = load_defaults();
5096
5097     # set environment variable useful inside network script
5098     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5099
5100     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5101
5102     my $forcemachine = $params->{forcemachine};
5103     my $forcecpu = $params->{forcecpu};
5104     if ($resume) {
5105         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5106         $forcemachine = $conf->{runningmachine};
5107         $forcecpu = $conf->{runningcpu};
5108         print "Resuming suspended VM\n";
5109     }
5110
5111     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5112         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5113
5114     my $migration_ip;
5115     my $get_migration_ip = sub {
5116         my ($nodename) = @_;
5117
5118         return $migration_ip if defined($migration_ip);
5119
5120         my $cidr = $migrate_opts->{network};
5121
5122         if (!defined($cidr)) {
5123             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5124             $cidr = $dc_conf->{migration}->{network};
5125         }
5126
5127         if (defined($cidr)) {
5128             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5129
5130             die "could not get IP: no address configured on local " .
5131                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5132
5133             die "could not get IP: multiple addresses configured on local " .
5134                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5135
5136             $migration_ip = @$ips[0];
5137         }
5138
5139         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5140             if !defined($migration_ip);
5141
5142         return $migration_ip;
5143     };
5144
5145     my $migrate_uri;
5146     if ($statefile) {
5147         if ($statefile eq 'tcp') {
5148             my $localip = "localhost";
5149             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5150             my $nodename = nodename();
5151
5152             if (!defined($migration_type)) {
5153                 if (defined($datacenterconf->{migration}->{type})) {
5154                     $migration_type = $datacenterconf->{migration}->{type};
5155                 } else {
5156                     $migration_type = 'secure';
5157                 }
5158             }
5159
5160             if ($migration_type eq 'insecure') {
5161                 $localip = $get_migration_ip->($nodename);
5162                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5163             }
5164
5165             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5166             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5167             $migrate_uri = "tcp:${localip}:${migrate_port}";
5168             push @$cmd, '-incoming', $migrate_uri;
5169             push @$cmd, '-S';
5170
5171         } elsif ($statefile eq 'unix') {
5172             # should be default for secure migrations as a ssh TCP forward
5173             # tunnel is not deterministic reliable ready and fails regurarly
5174             # to set up in time, so use UNIX socket forwards
5175             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5176             unlink $socket_addr;
5177
5178             $migrate_uri = "unix:$socket_addr";
5179
5180             push @$cmd, '-incoming', $migrate_uri;
5181             push @$cmd, '-S';
5182
5183         } elsif (-e $statefile) {
5184             push @$cmd, '-loadstate', $statefile;
5185         } else {
5186             my $statepath = PVE::Storage::path($storecfg, $statefile);
5187             push @$vollist, $statefile;
5188             push @$cmd, '-loadstate', $statepath;
5189         }
5190     } elsif ($params->{paused}) {
5191         push @$cmd, '-S';
5192     }
5193
5194     # host pci devices
5195     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5196       my $d = parse_hostpci($conf->{"hostpci$i"});
5197       next if !$d;
5198       my $pcidevices = $d->{pciid};
5199       foreach my $pcidevice (@$pcidevices) {
5200             my $pciid = $pcidevice->{id};
5201
5202             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5203             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5204             die "no pci device info for device '$pciid'\n" if !$info;
5205
5206             if ($d->{mdev}) {
5207                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5208                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5209             } else {
5210                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5211                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5212                 die "can't reset PCI device '$pciid'\n"
5213                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5214             }
5215       }
5216     }
5217
5218     PVE::Storage::activate_volumes($storecfg, $vollist);
5219
5220     eval {
5221         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5222             outfunc => sub {}, errfunc => sub {});
5223     };
5224     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5225     # timeout should be more than enough here...
5226     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5227
5228     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5229                                               : $defaults->{cpuunits};
5230
5231     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5232     my %run_params = (
5233         timeout => $statefile ? undef : $start_timeout,
5234         umask => 0077,
5235         noerr => 1,
5236     );
5237
5238     # when migrating, prefix QEMU output so other side can pick up any
5239     # errors that might occur and show the user
5240     if ($migratedfrom) {
5241         $run_params{quiet} = 1;
5242         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5243     }
5244
5245     my %properties = (
5246         Slice => 'qemu.slice',
5247         KillMode => 'none'
5248     );
5249
5250     if (PVE::CGroup::cgroup_mode() == 2) {
5251         $properties{CPUWeight} = $cpuunits;
5252     } else {
5253         $properties{CPUShares} = $cpuunits;
5254     }
5255
5256     if (my $cpulimit = $conf->{cpulimit}) {
5257         $properties{CPUQuota} = int($cpulimit * 100);
5258     }
5259     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5260
5261     my $run_qemu = sub {
5262         PVE::Tools::run_fork sub {
5263             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5264
5265             my $exitcode = run_command($cmd, %run_params);
5266             die "QEMU exited with code $exitcode\n" if $exitcode;
5267         };
5268     };
5269
5270     if ($conf->{hugepages}) {
5271
5272         my $code = sub {
5273             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5274             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5275
5276             PVE::QemuServer::Memory::hugepages_mount();
5277             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5278
5279             eval { $run_qemu->() };
5280             if (my $err = $@) {
5281                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5282                     if !$conf->{keephugepages};
5283                 die $err;
5284             }
5285
5286             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5287                 if !$conf->{keephugepages};
5288         };
5289         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5290
5291     } else {
5292         eval { $run_qemu->() };
5293     }
5294
5295     if (my $err = $@) {
5296         # deactivate volumes if start fails
5297         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5298         die "start failed: $err";
5299     }
5300
5301     print "migration listens on $migrate_uri\n" if $migrate_uri;
5302     $res->{migrate_uri} = $migrate_uri;
5303
5304     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5305         eval { mon_cmd($vmid, "cont"); };
5306         warn $@ if $@;
5307     }
5308
5309     #start nbd server for storage migration
5310     if (my $nbd = $migrate_opts->{nbd}) {
5311         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5312
5313         my $migrate_storage_uri;
5314         # nbd_protocol_version > 0 for unix socket support
5315         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5316             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5317             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5318             $migrate_storage_uri = "nbd:unix:$socket_path";
5319         } else {
5320             my $nodename = nodename();
5321             my $localip = $get_migration_ip->($nodename);
5322             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5323             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5324
5325             mon_cmd($vmid, "nbd-server-start", addr => {
5326                 type => 'inet',
5327                 data => {
5328                     host => "${localip}",
5329                     port => "${storage_migrate_port}",
5330                 },
5331             });
5332             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5333             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5334         }
5335
5336         $res->{migrate_storage_uri} = $migrate_storage_uri;
5337
5338         foreach my $opt (sort keys %$nbd) {
5339             my $drivestr = $nbd->{$opt}->{drivestr};
5340             my $volid = $nbd->{$opt}->{volid};
5341             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5342             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5343             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5344             print "re-using replicated volume: $opt - $volid\n"
5345                 if $nbd->{$opt}->{replicated};
5346
5347             $res->{drives}->{$opt} = $nbd->{$opt};
5348             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5349         }
5350     }
5351
5352     if ($migratedfrom) {
5353         eval {
5354             set_migration_caps($vmid);
5355         };
5356         warn $@ if $@;
5357
5358         if ($spice_port) {
5359             print "spice listens on port $spice_port\n";
5360             $res->{spice_port} = $spice_port;
5361             if ($migrate_opts->{spice_ticket}) {
5362                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5363                     $migrate_opts->{spice_ticket});
5364                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5365             }
5366         }
5367
5368     } else {
5369         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5370             if !$statefile && $conf->{balloon};
5371
5372         foreach my $opt (keys %$conf) {
5373             next if $opt !~  m/^net\d+$/;
5374             my $nicconf = parse_net($conf->{$opt});
5375             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5376         }
5377     }
5378
5379     mon_cmd($vmid, 'qom-set',
5380                 path => "machine/peripheral/balloon0",
5381                 property => "guest-stats-polling-interval",
5382                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5383
5384     if ($resume) {
5385         print "Resumed VM, removing state\n";
5386         if (my $vmstate = $conf->{vmstate}) {
5387             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5388             PVE::Storage::vdisk_free($storecfg, $vmstate);
5389         }
5390         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5391         PVE::QemuConfig->write_config($vmid, $conf);
5392     }
5393
5394     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5395
5396     return $res;
5397 }
5398
5399 sub vm_commandline {
5400     my ($storecfg, $vmid, $snapname) = @_;
5401
5402     my $conf = PVE::QemuConfig->load_config($vmid);
5403     my $forcemachine;
5404     my $forcecpu;
5405
5406     if ($snapname) {
5407         my $snapshot = $conf->{snapshots}->{$snapname};
5408         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5409
5410         # check for machine or CPU overrides in snapshot
5411         $forcemachine = $snapshot->{runningmachine};
5412         $forcecpu = $snapshot->{runningcpu};
5413
5414         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5415
5416         $conf = $snapshot;
5417     }
5418
5419     my $defaults = load_defaults();
5420
5421     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5422         $forcemachine, $forcecpu);
5423
5424     return PVE::Tools::cmd2string($cmd);
5425 }
5426
5427 sub vm_reset {
5428     my ($vmid, $skiplock) = @_;
5429
5430     PVE::QemuConfig->lock_config($vmid, sub {
5431
5432         my $conf = PVE::QemuConfig->load_config($vmid);
5433
5434         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5435
5436         mon_cmd($vmid, "system_reset");
5437     });
5438 }
5439
5440 sub get_vm_volumes {
5441     my ($conf) = @_;
5442
5443     my $vollist = [];
5444     foreach_volid($conf, sub {
5445         my ($volid, $attr) = @_;
5446
5447         return if $volid =~ m|^/|;
5448
5449         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5450         return if !$sid;
5451
5452         push @$vollist, $volid;
5453     });
5454
5455     return $vollist;
5456 }
5457
5458 sub vm_stop_cleanup {
5459     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5460
5461     eval {
5462
5463         if (!$keepActive) {
5464             my $vollist = get_vm_volumes($conf);
5465             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5466         }
5467
5468         foreach my $ext (qw(mon qmp pid vnc qga)) {
5469             unlink "/var/run/qemu-server/${vmid}.$ext";
5470         }
5471
5472         if ($conf->{ivshmem}) {
5473             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5474             # just delete it for now, VMs which have this already open do not
5475             # are affected, but new VMs will get a separated one. If this
5476             # becomes an issue we either add some sort of ref-counting or just
5477             # add a "don't delete on stop" flag to the ivshmem format.
5478             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5479         }
5480
5481         foreach my $key (keys %$conf) {
5482             next if $key !~ m/^hostpci(\d+)$/;
5483             my $hostpciindex = $1;
5484             my $d = parse_hostpci($conf->{$key});
5485             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5486
5487             foreach my $pci (@{$d->{pciid}}) {
5488                 my $pciid = $pci->{id};
5489                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5490             }
5491         }
5492
5493         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5494     };
5495     warn $@ if $@; # avoid errors - just warn
5496 }
5497
5498 # call only in locked context
5499 sub _do_vm_stop {
5500     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5501
5502     my $pid = check_running($vmid, $nocheck);
5503     return if !$pid;
5504
5505     my $conf;
5506     if (!$nocheck) {
5507         $conf = PVE::QemuConfig->load_config($vmid);
5508         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5509         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5510             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5511             $timeout = $opts->{down} if $opts->{down};
5512         }
5513         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5514     }
5515
5516     eval {
5517         if ($shutdown) {
5518             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5519                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5520             } else {
5521                 mon_cmd($vmid, "system_powerdown");
5522             }
5523         } else {
5524             mon_cmd($vmid, "quit");
5525         }
5526     };
5527     my $err = $@;
5528
5529     if (!$err) {
5530         $timeout = 60 if !defined($timeout);
5531
5532         my $count = 0;
5533         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5534             $count++;
5535             sleep 1;
5536         }
5537
5538         if ($count >= $timeout) {
5539             if ($force) {
5540                 warn "VM still running - terminating now with SIGTERM\n";
5541                 kill 15, $pid;
5542             } else {
5543                 die "VM quit/powerdown failed - got timeout\n";
5544             }
5545         } else {
5546             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5547             return;
5548         }
5549     } else {
5550        if (!check_running($vmid, $nocheck)) {
5551             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5552             return;
5553        }
5554        if ($force) {
5555             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5556             kill 15, $pid;
5557         } else {
5558             die "VM quit/powerdown failed\n";
5559         }
5560     }
5561
5562     # wait again
5563     $timeout = 10;
5564
5565     my $count = 0;
5566     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5567         $count++;
5568         sleep 1;
5569     }
5570
5571     if ($count >= $timeout) {
5572         warn "VM still running - terminating now with SIGKILL\n";
5573         kill 9, $pid;
5574         sleep 1;
5575     }
5576
5577     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5578 }
5579
5580 # Note: use $nocheck to skip tests if VM configuration file exists.
5581 # We need that when migration VMs to other nodes (files already moved)
5582 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5583 sub vm_stop {
5584     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5585
5586     $force = 1 if !defined($force) && !$shutdown;
5587
5588     if ($migratedfrom){
5589         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5590         kill 15, $pid if $pid;
5591         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5592         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5593         return;
5594     }
5595
5596     PVE::QemuConfig->lock_config($vmid, sub {
5597         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5598    });
5599 }
5600
5601 sub vm_reboot {
5602     my ($vmid, $timeout) = @_;
5603
5604     PVE::QemuConfig->lock_config($vmid, sub {
5605         eval {
5606
5607             # only reboot if running, as qmeventd starts it again on a stop event
5608             return if !check_running($vmid);
5609
5610             create_reboot_request($vmid);
5611
5612             my $storecfg = PVE::Storage::config();
5613             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5614
5615         };
5616         if (my $err = $@) {
5617             # avoid that the next normal shutdown will be confused for a reboot
5618             clear_reboot_request($vmid);
5619             die $err;
5620         }
5621    });
5622 }
5623
5624 # note: if using the statestorage parameter, the caller has to check privileges
5625 sub vm_suspend {
5626     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5627
5628     my $conf;
5629     my $path;
5630     my $storecfg;
5631     my $vmstate;
5632
5633     PVE::QemuConfig->lock_config($vmid, sub {
5634
5635         $conf = PVE::QemuConfig->load_config($vmid);
5636
5637         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5638         PVE::QemuConfig->check_lock($conf)
5639             if !($skiplock || $is_backing_up);
5640
5641         die "cannot suspend to disk during backup\n"
5642             if $is_backing_up && $includestate;
5643
5644         if ($includestate) {
5645             $conf->{lock} = 'suspending';
5646             my $date = strftime("%Y-%m-%d", localtime(time()));
5647             $storecfg = PVE::Storage::config();
5648             if (!$statestorage) {
5649                 $statestorage = find_vmstate_storage($conf, $storecfg);
5650                 # check permissions for the storage
5651                 my $rpcenv = PVE::RPCEnvironment::get();
5652                 if ($rpcenv->{type} ne 'cli') {
5653                     my $authuser = $rpcenv->get_user();
5654                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5655                 }
5656             }
5657
5658
5659             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5660                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5661             $path = PVE::Storage::path($storecfg, $vmstate);
5662             PVE::QemuConfig->write_config($vmid, $conf);
5663         } else {
5664             mon_cmd($vmid, "stop");
5665         }
5666     });
5667
5668     if ($includestate) {
5669         # save vm state
5670         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5671
5672         eval {
5673             set_migration_caps($vmid, 1);
5674             mon_cmd($vmid, "savevm-start", statefile => $path);
5675             for(;;) {
5676                 my $state = mon_cmd($vmid, "query-savevm");
5677                 if (!$state->{status}) {
5678                     die "savevm not active\n";
5679                 } elsif ($state->{status} eq 'active') {
5680                     sleep(1);
5681                     next;
5682                 } elsif ($state->{status} eq 'completed') {
5683                     print "State saved, quitting\n";
5684                     last;
5685                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5686                     die "query-savevm failed with error '$state->{error}'\n"
5687                 } else {
5688                     die "query-savevm returned status '$state->{status}'\n";
5689                 }
5690             }
5691         };
5692         my $err = $@;
5693
5694         PVE::QemuConfig->lock_config($vmid, sub {
5695             $conf = PVE::QemuConfig->load_config($vmid);
5696             if ($err) {
5697                 # cleanup, but leave suspending lock, to indicate something went wrong
5698                 eval {
5699                     mon_cmd($vmid, "savevm-end");
5700                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5701                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5702                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5703                     PVE::QemuConfig->write_config($vmid, $conf);
5704                 };
5705                 warn $@ if $@;
5706                 die $err;
5707             }
5708
5709             die "lock changed unexpectedly\n"
5710                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5711
5712             mon_cmd($vmid, "quit");
5713             $conf->{lock} = 'suspended';
5714             PVE::QemuConfig->write_config($vmid, $conf);
5715         });
5716     }
5717 }
5718
5719 sub vm_resume {
5720     my ($vmid, $skiplock, $nocheck) = @_;
5721
5722     PVE::QemuConfig->lock_config($vmid, sub {
5723         my $res = mon_cmd($vmid, 'query-status');
5724         my $resume_cmd = 'cont';
5725         my $reset = 0;
5726
5727         if ($res->{status}) {
5728             return if $res->{status} eq 'running'; # job done, go home
5729             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5730             $reset = 1 if $res->{status} eq 'shutdown';
5731         }
5732
5733         if (!$nocheck) {
5734
5735             my $conf = PVE::QemuConfig->load_config($vmid);
5736
5737             PVE::QemuConfig->check_lock($conf)
5738                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5739         }
5740
5741         if ($reset) {
5742             # required if a VM shuts down during a backup and we get a resume
5743             # request before the backup finishes for example
5744             mon_cmd($vmid, "system_reset");
5745         }
5746         mon_cmd($vmid, $resume_cmd);
5747     });
5748 }
5749
5750 sub vm_sendkey {
5751     my ($vmid, $skiplock, $key) = @_;
5752
5753     PVE::QemuConfig->lock_config($vmid, sub {
5754
5755         my $conf = PVE::QemuConfig->load_config($vmid);
5756
5757         # there is no qmp command, so we use the human monitor command
5758         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5759         die $res if $res ne '';
5760     });
5761 }
5762
5763 # vzdump restore implementaion
5764
5765 sub tar_archive_read_firstfile {
5766     my $archive = shift;
5767
5768     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5769
5770     # try to detect archive type first
5771     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5772         die "unable to open file '$archive'\n";
5773     my $firstfile = <$fh>;
5774     kill 15, $pid;
5775     close $fh;
5776
5777     die "ERROR: archive contaions no data\n" if !$firstfile;
5778     chomp $firstfile;
5779
5780     return $firstfile;
5781 }
5782
5783 sub tar_restore_cleanup {
5784     my ($storecfg, $statfile) = @_;
5785
5786     print STDERR "starting cleanup\n";
5787
5788     if (my $fd = IO::File->new($statfile, "r")) {
5789         while (defined(my $line = <$fd>)) {
5790             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5791                 my $volid = $2;
5792                 eval {
5793                     if ($volid =~ m|^/|) {
5794                         unlink $volid || die 'unlink failed\n';
5795                     } else {
5796                         PVE::Storage::vdisk_free($storecfg, $volid);
5797                     }
5798                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5799                 };
5800                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5801             } else {
5802                 print STDERR "unable to parse line in statfile - $line";
5803             }
5804         }
5805         $fd->close();
5806     }
5807 }
5808
5809 sub restore_file_archive {
5810     my ($archive, $vmid, $user, $opts) = @_;
5811
5812     return restore_vma_archive($archive, $vmid, $user, $opts)
5813         if $archive eq '-';
5814
5815     my $info = PVE::Storage::archive_info($archive);
5816     my $format = $opts->{format} // $info->{format};
5817     my $comp = $info->{compression};
5818
5819     # try to detect archive format
5820     if ($format eq 'tar') {
5821         return restore_tar_archive($archive, $vmid, $user, $opts);
5822     } else {
5823         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5824     }
5825 }
5826
5827 # hepler to remove disks that will not be used after restore
5828 my $restore_cleanup_oldconf = sub {
5829     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5830
5831     PVE::QemuConfig->foreach_volume($oldconf, sub {
5832         my ($ds, $drive) = @_;
5833
5834         return if drive_is_cdrom($drive, 1);
5835
5836         my $volid = $drive->{file};
5837         return if !$volid || $volid =~ m|^/|;
5838
5839         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5840         return if !$path || !$owner || ($owner != $vmid);
5841
5842         # Note: only delete disk we want to restore
5843         # other volumes will become unused
5844         if ($virtdev_hash->{$ds}) {
5845             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5846             if (my $err = $@) {
5847                 warn $err;
5848             }
5849         }
5850     });
5851
5852     # delete vmstate files, after the restore we have no snapshots anymore
5853     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5854         my $snap = $oldconf->{snapshots}->{$snapname};
5855         if ($snap->{vmstate}) {
5856             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5857             if (my $err = $@) {
5858                 warn $err;
5859             }
5860         }
5861     }
5862 };
5863
5864 # Helper to parse vzdump backup device hints
5865 #
5866 # $rpcenv: Environment, used to ckeck storage permissions
5867 # $user: User ID, to check storage permissions
5868 # $storecfg: Storage configuration
5869 # $fh: the file handle for reading the configuration
5870 # $devinfo: should contain device sizes for all backu-up'ed devices
5871 # $options: backup options (pool, default storage)
5872 #
5873 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5874 my $parse_backup_hints = sub {
5875     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5876
5877     my $virtdev_hash = {};
5878
5879     while (defined(my $line = <$fh>)) {
5880         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5881             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5882             die "archive does not contain data for drive '$virtdev'\n"
5883                 if !$devinfo->{$devname};
5884
5885             if (defined($options->{storage})) {
5886                 $storeid = $options->{storage} || 'local';
5887             } elsif (!$storeid) {
5888                 $storeid = 'local';
5889             }
5890             $format = 'raw' if !$format;
5891             $devinfo->{$devname}->{devname} = $devname;
5892             $devinfo->{$devname}->{virtdev} = $virtdev;
5893             $devinfo->{$devname}->{format} = $format;
5894             $devinfo->{$devname}->{storeid} = $storeid;
5895
5896             # check permission on storage
5897             my $pool = $options->{pool}; # todo: do we need that?
5898             if ($user ne 'root@pam') {
5899                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5900             }
5901
5902             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5903         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5904             my $virtdev = $1;
5905             my $drive = parse_drive($virtdev, $2);
5906             if (drive_is_cloudinit($drive)) {
5907                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5908                 $storeid = $options->{storage} if defined ($options->{storage});
5909                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5910                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5911
5912                 $virtdev_hash->{$virtdev} = {
5913                     format => $format,
5914                     storeid => $storeid,
5915                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5916                     is_cloudinit => 1,
5917                 };
5918             }
5919         }
5920     }
5921
5922     return $virtdev_hash;
5923 };
5924
5925 # Helper to allocate and activate all volumes required for a restore
5926 #
5927 # $storecfg: Storage configuration
5928 # $virtdev_hash: as returned by parse_backup_hints()
5929 #
5930 # Returns: { $virtdev => $volid }
5931 my $restore_allocate_devices = sub {
5932     my ($storecfg, $virtdev_hash, $vmid) = @_;
5933
5934     my $map = {};
5935     foreach my $virtdev (sort keys %$virtdev_hash) {
5936         my $d = $virtdev_hash->{$virtdev};
5937         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5938         my $storeid = $d->{storeid};
5939         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5940
5941         # test if requested format is supported
5942         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5943         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5944         $d->{format} = $defFormat if !$supported;
5945
5946         my $name;
5947         if ($d->{is_cloudinit}) {
5948             $name = "vm-$vmid-cloudinit";
5949             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5950             if ($scfg->{path}) {
5951                 $name .= ".$d->{format}";
5952             }
5953         }
5954
5955         my $volid = PVE::Storage::vdisk_alloc(
5956             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5957
5958         print STDERR "new volume ID is '$volid'\n";
5959         $d->{volid} = $volid;
5960
5961         PVE::Storage::activate_volumes($storecfg, [$volid]);
5962
5963         $map->{$virtdev} = $volid;
5964     }
5965
5966     return $map;
5967 };
5968
5969 my $restore_update_config_line = sub {
5970     my ($cookie, $vmid, $map, $line, $unique) = @_;
5971
5972     return '' if $line =~ m/^\#qmdump\#/;
5973     return '' if $line =~ m/^\#vzdump\#/;
5974     return '' if $line =~ m/^lock:/;
5975     return '' if $line =~ m/^unused\d+:/;
5976     return '' if $line =~ m/^parent:/;
5977
5978     my $res = '';
5979
5980     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5981     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5982         # try to convert old 1.X settings
5983         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5984         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5985             my ($model, $macaddr) = split(/\=/, $devconfig);
5986             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5987             my $net = {
5988                 model => $model,
5989                 bridge => "vmbr$ind",
5990                 macaddr => $macaddr,
5991             };
5992             my $netstr = print_net($net);
5993
5994             $res .= "net$cookie->{netcount}: $netstr\n";
5995             $cookie->{netcount}++;
5996         }
5997     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5998         my ($id, $netstr) = ($1, $2);
5999         my $net = parse_net($netstr);
6000         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6001         $netstr = print_net($net);
6002         $res .= "$id: $netstr\n";
6003     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6004         my $virtdev = $1;
6005         my $value = $3;
6006         my $di = parse_drive($virtdev, $value);
6007         if (defined($di->{backup}) && !$di->{backup}) {
6008             $res .= "#$line";
6009         } elsif ($map->{$virtdev}) {
6010             delete $di->{format}; # format can change on restore
6011             $di->{file} = $map->{$virtdev};
6012             $value = print_drive($di);
6013             $res .= "$virtdev: $value\n";
6014         } else {
6015             $res .= $line;
6016         }
6017     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6018         my $vmgenid = $1;
6019         if ($vmgenid ne '0') {
6020             # always generate a new vmgenid if there was a valid one setup
6021             $vmgenid = generate_uuid();
6022         }
6023         $res .= "vmgenid: $vmgenid\n";
6024     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6025         my ($uuid, $uuid_str);
6026         UUID::generate($uuid);
6027         UUID::unparse($uuid, $uuid_str);
6028         my $smbios1 = parse_smbios1($2);
6029         $smbios1->{uuid} = $uuid_str;
6030         $res .= $1.print_smbios1($smbios1)."\n";
6031     } else {
6032         $res .= $line;
6033     }
6034
6035     return $res;
6036 };
6037
6038 my $restore_deactivate_volumes = sub {
6039     my ($storecfg, $devinfo) = @_;
6040
6041     my $vollist = [];
6042     foreach my $devname (keys %$devinfo) {
6043         my $volid = $devinfo->{$devname}->{volid};
6044         push @$vollist, $volid if $volid;
6045     }
6046
6047     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6048 };
6049
6050 my $restore_destroy_volumes = sub {
6051     my ($storecfg, $devinfo) = @_;
6052
6053     foreach my $devname (keys %$devinfo) {
6054         my $volid = $devinfo->{$devname}->{volid};
6055         next if !$volid;
6056         eval {
6057             if ($volid =~ m|^/|) {
6058                 unlink $volid || die 'unlink failed\n';
6059             } else {
6060                 PVE::Storage::vdisk_free($storecfg, $volid);
6061             }
6062             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6063         };
6064         print STDERR "unable to cleanup '$volid' - $@" if $@;
6065     }
6066 };
6067
6068 sub scan_volids {
6069     my ($cfg, $vmid) = @_;
6070
6071     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6072
6073     my $volid_hash = {};
6074     foreach my $storeid (keys %$info) {
6075         foreach my $item (@{$info->{$storeid}}) {
6076             next if !($item->{volid} && $item->{size});
6077             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6078             $volid_hash->{$item->{volid}} = $item;
6079         }
6080     }
6081
6082     return $volid_hash;
6083 }
6084
6085 sub update_disk_config {
6086     my ($vmid, $conf, $volid_hash) = @_;
6087
6088     my $changes;
6089     my $prefix = "VM $vmid";
6090
6091     # used and unused disks
6092     my $referenced = {};
6093
6094     # Note: it is allowed to define multiple storages with same path (alias), so
6095     # we need to check both 'volid' and real 'path' (two different volid can point
6096     # to the same path).
6097
6098     my $referencedpath = {};
6099
6100     # update size info
6101     PVE::QemuConfig->foreach_volume($conf, sub {
6102         my ($opt, $drive) = @_;
6103
6104         my $volid = $drive->{file};
6105         return if !$volid;
6106         my $volume = $volid_hash->{$volid};
6107
6108         # mark volid as "in-use" for next step
6109         $referenced->{$volid} = 1;
6110         if ($volume && (my $path = $volume->{path})) {
6111             $referencedpath->{$path} = 1;
6112         }
6113
6114         return if drive_is_cdrom($drive);
6115         return if !$volume;
6116
6117         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6118         if (defined($updated)) {
6119             $changes = 1;
6120             $conf->{$opt} = print_drive($updated);
6121             print "$prefix ($opt): $msg\n";
6122         }
6123     });
6124
6125     # remove 'unusedX' entry if volume is used
6126     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6127         my ($opt, $drive) = @_;
6128
6129         my $volid = $drive->{file};
6130         return if !$volid;
6131
6132         my $path;
6133         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6134         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6135             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6136             $changes = 1;
6137             delete $conf->{$opt};
6138         }
6139
6140         $referenced->{$volid} = 1;
6141         $referencedpath->{$path} = 1 if $path;
6142     });
6143
6144     foreach my $volid (sort keys %$volid_hash) {
6145         next if $volid =~ m/vm-$vmid-state-/;
6146         next if $referenced->{$volid};
6147         my $path = $volid_hash->{$volid}->{path};
6148         next if !$path; # just to be sure
6149         next if $referencedpath->{$path};
6150         $changes = 1;
6151         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6152         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6153         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6154     }
6155
6156     return $changes;
6157 }
6158
6159 sub rescan {
6160     my ($vmid, $nolock, $dryrun) = @_;
6161
6162     my $cfg = PVE::Storage::config();
6163
6164     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6165     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6166     foreach my $stor (keys %{$cfg->{ids}}) {
6167         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6168     }
6169
6170     print "rescan volumes...\n";
6171     my $volid_hash = scan_volids($cfg, $vmid);
6172
6173     my $updatefn =  sub {
6174         my ($vmid) = @_;
6175
6176         my $conf = PVE::QemuConfig->load_config($vmid);
6177
6178         PVE::QemuConfig->check_lock($conf);
6179
6180         my $vm_volids = {};
6181         foreach my $volid (keys %$volid_hash) {
6182             my $info = $volid_hash->{$volid};
6183             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6184         }
6185
6186         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6187
6188         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6189     };
6190
6191     if (defined($vmid)) {
6192         if ($nolock) {
6193             &$updatefn($vmid);
6194         } else {
6195             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6196         }
6197     } else {
6198         my $vmlist = config_list();
6199         foreach my $vmid (keys %$vmlist) {
6200             if ($nolock) {
6201                 &$updatefn($vmid);
6202             } else {
6203                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6204             }
6205         }
6206     }
6207 }
6208
6209 sub restore_proxmox_backup_archive {
6210     my ($archive, $vmid, $user, $options) = @_;
6211
6212     my $storecfg = PVE::Storage::config();
6213
6214     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6215     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6216
6217     my $fingerprint = $scfg->{fingerprint};
6218     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6219
6220     my $repo = PVE::PBSClient::get_repository($scfg);
6221
6222     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6223     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6224     local $ENV{PBS_PASSWORD} = $password;
6225     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6226
6227     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6228         PVE::Storage::parse_volname($storecfg, $archive);
6229
6230     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6231
6232     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6233
6234     my $tmpdir = "/var/tmp/vzdumptmp$$";
6235     rmtree $tmpdir;
6236     mkpath $tmpdir;
6237
6238     my $conffile = PVE::QemuConfig->config_file($vmid);
6239      # disable interrupts (always do cleanups)
6240     local $SIG{INT} =
6241         local $SIG{TERM} =
6242         local $SIG{QUIT} =
6243         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6244
6245     # Note: $oldconf is undef if VM does not exists
6246     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6247     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6248     my $new_conf_raw = '';
6249
6250     my $rpcenv = PVE::RPCEnvironment::get();
6251     my $devinfo = {};
6252
6253     eval {
6254         # enable interrupts
6255         local $SIG{INT} =
6256             local $SIG{TERM} =
6257             local $SIG{QUIT} =
6258             local $SIG{HUP} =
6259             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6260
6261         my $cfgfn = "$tmpdir/qemu-server.conf";
6262         my $firewall_config_fn = "$tmpdir/fw.conf";
6263         my $index_fn = "$tmpdir/index.json";
6264
6265         my $cmd = "restore";
6266
6267         my $param = [$pbs_backup_name, "index.json", $index_fn];
6268         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6269         my $index = PVE::Tools::file_get_contents($index_fn);
6270         $index = decode_json($index);
6271
6272         # print Dumper($index);
6273         foreach my $info (@{$index->{files}}) {
6274             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6275                 my $devname = $1;
6276                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6277                     $devinfo->{$devname}->{size} = $1;
6278                 } else {
6279                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6280                 }
6281             }
6282         }
6283
6284         my $is_qemu_server_backup = scalar(
6285             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6286         );
6287         if (!$is_qemu_server_backup) {
6288             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6289         }
6290         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6291
6292         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6293         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6294
6295         if ($has_firewall_config) {
6296             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6297             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6298
6299             my $pve_firewall_dir = '/etc/pve/firewall';
6300             mkdir $pve_firewall_dir; # make sure the dir exists
6301             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6302         }
6303
6304         my $fh = IO::File->new($cfgfn, "r") ||
6305             die "unable to read qemu-server.conf - $!\n";
6306
6307         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6308
6309         # fixme: rate limit?
6310
6311         # create empty/temp config
6312         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6313
6314         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6315
6316         # allocate volumes
6317         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6318
6319         if (!$options->{live}) {
6320             foreach my $virtdev (sort keys %$virtdev_hash) {
6321                 my $d = $virtdev_hash->{$virtdev};
6322                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6323
6324                 my $volid = $d->{volid};
6325
6326                 my $path = PVE::Storage::path($storecfg, $volid);
6327
6328                 my $pbs_restore_cmd = [
6329                     '/usr/bin/pbs-restore',
6330                     '--repository', $repo,
6331                     $pbs_backup_name,
6332                     "$d->{devname}.img.fidx",
6333                     $path,
6334                     '--verbose',
6335                     ];
6336
6337                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6338                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6339
6340                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6341                     push @$pbs_restore_cmd, '--skip-zero';
6342                 }
6343
6344                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6345                 print "restore proxmox backup image: $dbg_cmdstring\n";
6346                 run_command($pbs_restore_cmd);
6347             }
6348         }
6349
6350         $fh->seek(0, 0) || die "seek failed - $!\n";
6351
6352         my $cookie = { netcount => 0 };
6353         while (defined(my $line = <$fh>)) {
6354             $new_conf_raw .= $restore_update_config_line->(
6355                 $cookie,
6356                 $vmid,
6357                 $map,
6358                 $line,
6359                 $options->{unique},
6360             );
6361         }
6362
6363         $fh->close();
6364     };
6365     my $err = $@;
6366
6367     if ($err || !$options->{live}) {
6368         $restore_deactivate_volumes->($storecfg, $devinfo);
6369     }
6370
6371     rmtree $tmpdir;
6372
6373     if ($err) {
6374         $restore_destroy_volumes->($storecfg, $devinfo);
6375         die $err;
6376     }
6377
6378     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6379
6380     PVE::Cluster::cfs_update(); # make sure we read new file
6381
6382     eval { rescan($vmid, 1); };
6383     warn $@ if $@;
6384
6385     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6386
6387     if ($options->{live}) {
6388         eval {
6389             # enable interrupts
6390             local $SIG{INT} =
6391                 local $SIG{TERM} =
6392                 local $SIG{QUIT} =
6393                 local $SIG{HUP} =
6394                 local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6395
6396             my $conf = PVE::QemuConfig->load_config($vmid);
6397             die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6398
6399             pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6400         };
6401
6402         $err = $@;
6403         if ($err) {
6404             warn "destroying partially live-restored VM, all temporary data will be lost!\n";
6405             $restore_deactivate_volumes->($storecfg, $devinfo);
6406             $restore_destroy_volumes->($storecfg, $devinfo);
6407             PVE::QemuConfig->destroy_config($vmid);
6408             die $err;
6409         }
6410     }
6411 }
6412
6413 sub pbs_live_restore {
6414     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6415
6416     print "Starting VM for live-restore\n";
6417
6418     my $pbs_backing = {};
6419     for my $ds (keys %$restored_disks) {
6420         $ds =~ m/^drive-(.*)$/;
6421         $pbs_backing->{$1} = {
6422             repository => $repo,
6423             snapshot => $snap,
6424             archive => "$ds.img.fidx",
6425         };
6426         $pbs_backing->{$1}->{keyfile} = $keyfile if -e $keyfile;
6427     }
6428
6429     my $drives_streamed = 0;
6430     eval {
6431         # make sure HA doesn't interrupt our restore by stopping the VM
6432         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6433             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6434         }
6435
6436         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6437         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6438         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6439
6440         my $qmeventd_fd = register_qmeventd_handle($vmid);
6441
6442         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6443         # this will effectively collapse the backing image chain consisting of
6444         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6445         # removes itself once all backing images vanish with 'auto-remove=on')
6446         my $jobs = {};
6447         for my $ds (sort keys %$restored_disks) {
6448             my $job_id = "restore-$ds";
6449             mon_cmd($vmid, 'block-stream',
6450                 'job-id' => $job_id,
6451                 device => "$ds",
6452             );
6453             $jobs->{$job_id} = {};
6454         }
6455
6456         mon_cmd($vmid, 'cont');
6457         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6458
6459         print "restore-drive jobs finished successfully, removing all tracking block devices"
6460             ." to disconnect from Proxmox Backup Server\n";
6461
6462         for my $ds (sort keys %$restored_disks) {
6463             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6464         }
6465
6466         close($qmeventd_fd);
6467     };
6468
6469     my $err = $@;
6470
6471     if ($err) {
6472         warn "An error occured during live-restore: $err\n";
6473         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6474         die "live-restore failed\n";
6475     }
6476 }
6477
6478 sub restore_vma_archive {
6479     my ($archive, $vmid, $user, $opts, $comp) = @_;
6480
6481     my $readfrom = $archive;
6482
6483     my $cfg = PVE::Storage::config();
6484     my $commands = [];
6485     my $bwlimit = $opts->{bwlimit};
6486
6487     my $dbg_cmdstring = '';
6488     my $add_pipe = sub {
6489         my ($cmd) = @_;
6490         push @$commands, $cmd;
6491         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6492         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6493         $readfrom = '-';
6494     };
6495
6496     my $input = undef;
6497     if ($archive eq '-') {
6498         $input = '<&STDIN';
6499     } else {
6500         # If we use a backup from a PVE defined storage we also consider that
6501         # storage's rate limit:
6502         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6503         if (defined($volid)) {
6504             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6505             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6506             if ($readlimit) {
6507                 print STDERR "applying read rate limit: $readlimit\n";
6508                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6509                 $add_pipe->($cstream);
6510             }
6511         }
6512     }
6513
6514     if ($comp) {
6515         my $info = PVE::Storage::decompressor_info('vma', $comp);
6516         my $cmd = $info->{decompressor};
6517         push @$cmd, $readfrom;
6518         $add_pipe->($cmd);
6519     }
6520
6521     my $tmpdir = "/var/tmp/vzdumptmp$$";
6522     rmtree $tmpdir;
6523
6524     # disable interrupts (always do cleanups)
6525     local $SIG{INT} =
6526         local $SIG{TERM} =
6527         local $SIG{QUIT} =
6528         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6529
6530     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6531     POSIX::mkfifo($mapfifo, 0600);
6532     my $fifofh;
6533     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6534
6535     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6536
6537     my $oldtimeout;
6538     my $timeout = 5;
6539
6540     my $devinfo = {};
6541
6542     my $rpcenv = PVE::RPCEnvironment::get();
6543
6544     my $conffile = PVE::QemuConfig->config_file($vmid);
6545
6546     # Note: $oldconf is undef if VM does not exist
6547     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6548     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6549     my $new_conf_raw = '';
6550
6551     my %storage_limits;
6552
6553     my $print_devmap = sub {
6554         my $cfgfn = "$tmpdir/qemu-server.conf";
6555
6556         # we can read the config - that is already extracted
6557         my $fh = IO::File->new($cfgfn, "r") ||
6558             die "unable to read qemu-server.conf - $!\n";
6559
6560         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6561         if (-f $fwcfgfn) {
6562             my $pve_firewall_dir = '/etc/pve/firewall';
6563             mkdir $pve_firewall_dir; # make sure the dir exists
6564             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6565         }
6566
6567         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6568
6569         foreach my $info (values %{$virtdev_hash}) {
6570             my $storeid = $info->{storeid};
6571             next if defined($storage_limits{$storeid});
6572
6573             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6574             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6575             $storage_limits{$storeid} = $limit * 1024;
6576         }
6577
6578         foreach my $devname (keys %$devinfo) {
6579             die "found no device mapping information for device '$devname'\n"
6580                 if !$devinfo->{$devname}->{virtdev};
6581         }
6582
6583         # create empty/temp config
6584         if ($oldconf) {
6585             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6586             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6587         }
6588
6589         # allocate volumes
6590         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6591
6592         # print restore information to $fifofh
6593         foreach my $virtdev (sort keys %$virtdev_hash) {
6594             my $d = $virtdev_hash->{$virtdev};
6595             next if $d->{is_cloudinit}; # no need to restore cloudinit
6596
6597             my $storeid = $d->{storeid};
6598             my $volid = $d->{volid};
6599
6600             my $map_opts = '';
6601             if (my $limit = $storage_limits{$storeid}) {
6602                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6603             }
6604
6605             my $write_zeros = 1;
6606             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6607                 $write_zeros = 0;
6608             }
6609
6610             my $path = PVE::Storage::path($cfg, $volid);
6611
6612             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6613
6614             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6615         }
6616
6617         $fh->seek(0, 0) || die "seek failed - $!\n";
6618
6619         my $cookie = { netcount => 0 };
6620         while (defined(my $line = <$fh>)) {
6621             $new_conf_raw .= $restore_update_config_line->(
6622                 $cookie,
6623                 $vmid,
6624                 $map,
6625                 $line,
6626                 $opts->{unique},
6627             );
6628         }
6629
6630         $fh->close();
6631     };
6632
6633     eval {
6634         # enable interrupts
6635         local $SIG{INT} =
6636             local $SIG{TERM} =
6637             local $SIG{QUIT} =
6638             local $SIG{HUP} =
6639             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6640         local $SIG{ALRM} = sub { die "got timeout\n"; };
6641
6642         $oldtimeout = alarm($timeout);
6643
6644         my $parser = sub {
6645             my $line = shift;
6646
6647             print "$line\n";
6648
6649             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6650                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6651                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6652             } elsif ($line =~ m/^CTIME: /) {
6653                 # we correctly received the vma config, so we can disable
6654                 # the timeout now for disk allocation (set to 10 minutes, so
6655                 # that we always timeout if something goes wrong)
6656                 alarm(600);
6657                 &$print_devmap();
6658                 print $fifofh "done\n";
6659                 my $tmp = $oldtimeout || 0;
6660                 $oldtimeout = undef;
6661                 alarm($tmp);
6662                 close($fifofh);
6663                 $fifofh = undef;
6664             }
6665         };
6666
6667         print "restore vma archive: $dbg_cmdstring\n";
6668         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6669     };
6670     my $err = $@;
6671
6672     alarm($oldtimeout) if $oldtimeout;
6673
6674     $restore_deactivate_volumes->($cfg, $devinfo);
6675
6676     close($fifofh) if $fifofh;
6677     unlink $mapfifo;
6678     rmtree $tmpdir;
6679
6680     if ($err) {
6681         $restore_destroy_volumes->($cfg, $devinfo);
6682         die $err;
6683     }
6684
6685     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6686
6687     PVE::Cluster::cfs_update(); # make sure we read new file
6688
6689     eval { rescan($vmid, 1); };
6690     warn $@ if $@;
6691
6692     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6693 }
6694
6695 sub restore_tar_archive {
6696     my ($archive, $vmid, $user, $opts) = @_;
6697
6698     if ($archive ne '-') {
6699         my $firstfile = tar_archive_read_firstfile($archive);
6700         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6701             if $firstfile ne 'qemu-server.conf';
6702     }
6703
6704     my $storecfg = PVE::Storage::config();
6705
6706     # avoid zombie disks when restoring over an existing VM -> cleanup first
6707     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6708     # skiplock=1 because qmrestore has set the 'create' lock itself already
6709     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6710     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6711
6712     my $tocmd = "/usr/lib/qemu-server/qmextract";
6713
6714     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6715     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6716     $tocmd .= ' --prealloc' if $opts->{prealloc};
6717     $tocmd .= ' --info' if $opts->{info};
6718
6719     # tar option "xf" does not autodetect compression when read from STDIN,
6720     # so we pipe to zcat
6721     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6722         PVE::Tools::shellquote("--to-command=$tocmd");
6723
6724     my $tmpdir = "/var/tmp/vzdumptmp$$";
6725     mkpath $tmpdir;
6726
6727     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6728     local $ENV{VZDUMP_VMID} = $vmid;
6729     local $ENV{VZDUMP_USER} = $user;
6730
6731     my $conffile = PVE::QemuConfig->config_file($vmid);
6732     my $new_conf_raw = '';
6733
6734     # disable interrupts (always do cleanups)
6735     local $SIG{INT} =
6736         local $SIG{TERM} =
6737         local $SIG{QUIT} =
6738         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6739
6740     eval {
6741         # enable interrupts
6742         local $SIG{INT} =
6743             local $SIG{TERM} =
6744             local $SIG{QUIT} =
6745             local $SIG{HUP} =
6746             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6747
6748         if ($archive eq '-') {
6749             print "extracting archive from STDIN\n";
6750             run_command($cmd, input => "<&STDIN");
6751         } else {
6752             print "extracting archive '$archive'\n";
6753             run_command($cmd);
6754         }
6755
6756         return if $opts->{info};
6757
6758         # read new mapping
6759         my $map = {};
6760         my $statfile = "$tmpdir/qmrestore.stat";
6761         if (my $fd = IO::File->new($statfile, "r")) {
6762             while (defined (my $line = <$fd>)) {
6763                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6764                     $map->{$1} = $2 if $1;
6765                 } else {
6766                     print STDERR "unable to parse line in statfile - $line\n";
6767                 }
6768             }
6769             $fd->close();
6770         }
6771
6772         my $confsrc = "$tmpdir/qemu-server.conf";
6773
6774         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6775
6776         my $cookie = { netcount => 0 };
6777         while (defined (my $line = <$srcfd>)) {
6778             $new_conf_raw .= $restore_update_config_line->(
6779                 $cookie,
6780                 $vmid,
6781                 $map,
6782                 $line,
6783                 $opts->{unique},
6784             );
6785         }
6786
6787         $srcfd->close();
6788     };
6789     if (my $err = $@) {
6790         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6791         die $err;
6792     }
6793
6794     rmtree $tmpdir;
6795
6796     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6797
6798     PVE::Cluster::cfs_update(); # make sure we read new file
6799
6800     eval { rescan($vmid, 1); };
6801     warn $@ if $@;
6802 };
6803
6804 sub foreach_storage_used_by_vm {
6805     my ($conf, $func) = @_;
6806
6807     my $sidhash = {};
6808
6809     PVE::QemuConfig->foreach_volume($conf, sub {
6810         my ($ds, $drive) = @_;
6811         return if drive_is_cdrom($drive);
6812
6813         my $volid = $drive->{file};
6814
6815         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6816         $sidhash->{$sid} = $sid if $sid;
6817     });
6818
6819     foreach my $sid (sort keys %$sidhash) {
6820         &$func($sid);
6821     }
6822 }
6823
6824 my $qemu_snap_storage = {
6825     rbd => 1,
6826 };
6827 sub do_snapshots_with_qemu {
6828     my ($storecfg, $volid) = @_;
6829
6830     my $storage_name = PVE::Storage::parse_volume_id($volid);
6831     my $scfg = $storecfg->{ids}->{$storage_name};
6832     die "could not find storage '$storage_name'\n" if !defined($scfg);
6833
6834     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6835         return 1;
6836     }
6837
6838     if ($volid =~ m/\.(qcow2|qed)$/){
6839         return 1;
6840     }
6841
6842     return;
6843 }
6844
6845 sub qga_check_running {
6846     my ($vmid, $nowarn) = @_;
6847
6848     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6849     if ($@) {
6850         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6851         return 0;
6852     }
6853     return 1;
6854 }
6855
6856 sub template_create {
6857     my ($vmid, $conf, $disk) = @_;
6858
6859     my $storecfg = PVE::Storage::config();
6860
6861     PVE::QemuConfig->foreach_volume($conf, sub {
6862         my ($ds, $drive) = @_;
6863
6864         return if drive_is_cdrom($drive);
6865         return if $disk && $ds ne $disk;
6866
6867         my $volid = $drive->{file};
6868         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6869
6870         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6871         $drive->{file} = $voliddst;
6872         $conf->{$ds} = print_drive($drive);
6873         PVE::QemuConfig->write_config($vmid, $conf);
6874     });
6875 }
6876
6877 sub convert_iscsi_path {
6878     my ($path) = @_;
6879
6880     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6881         my $portal = $1;
6882         my $target = $2;
6883         my $lun = $3;
6884
6885         my $initiator_name = get_initiator_name();
6886
6887         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6888                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6889     }
6890
6891     die "cannot convert iscsi path '$path', unkown format\n";
6892 }
6893
6894 sub qemu_img_convert {
6895     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6896
6897     my $storecfg = PVE::Storage::config();
6898     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6899     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6900
6901     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6902
6903     my $cachemode;
6904     my $src_path;
6905     my $src_is_iscsi = 0;
6906     my $src_format;
6907
6908     if ($src_storeid) {
6909         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6910         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6911         $src_format = qemu_img_format($src_scfg, $src_volname);
6912         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6913         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6914         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6915     } elsif (-f $src_volid) {
6916         $src_path = $src_volid;
6917         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6918             $src_format = $1;
6919         }
6920     }
6921
6922     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6923
6924     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6925     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6926     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6927     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6928
6929     my $cmd = [];
6930     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6931     push @$cmd, '-l', "snapshot.name=$snapname"
6932         if $snapname && $src_format && $src_format eq "qcow2";
6933     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6934     push @$cmd, '-T', $cachemode if defined($cachemode);
6935
6936     if ($src_is_iscsi) {
6937         push @$cmd, '--image-opts';
6938         $src_path = convert_iscsi_path($src_path);
6939     } elsif ($src_format) {
6940         push @$cmd, '-f', $src_format;
6941     }
6942
6943     if ($dst_is_iscsi) {
6944         push @$cmd, '--target-image-opts';
6945         $dst_path = convert_iscsi_path($dst_path);
6946     } else {
6947         push @$cmd, '-O', $dst_format;
6948     }
6949
6950     push @$cmd, $src_path;
6951
6952     if (!$dst_is_iscsi && $is_zero_initialized) {
6953         push @$cmd, "zeroinit:$dst_path";
6954     } else {
6955         push @$cmd, $dst_path;
6956     }
6957
6958     my $parser = sub {
6959         my $line = shift;
6960         if($line =~ m/\((\S+)\/100\%\)/){
6961             my $percent = $1;
6962             my $transferred = int($size * $percent / 100);
6963             my $remaining = $size - $transferred;
6964
6965             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6966         }
6967
6968     };
6969
6970     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6971     my $err = $@;
6972     die "copy failed: $err" if $err;
6973 }
6974
6975 sub qemu_img_format {
6976     my ($scfg, $volname) = @_;
6977
6978     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6979         return $1;
6980     } else {
6981         return "raw";
6982     }
6983 }
6984
6985 sub qemu_drive_mirror {
6986     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6987
6988     $jobs = {} if !$jobs;
6989
6990     my $qemu_target;
6991     my $format;
6992     $jobs->{"drive-$drive"} = {};
6993
6994     if ($dst_volid =~ /^nbd:/) {
6995         $qemu_target = $dst_volid;
6996         $format = "nbd";
6997     } else {
6998         my $storecfg = PVE::Storage::config();
6999         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7000
7001         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7002
7003         $format = qemu_img_format($dst_scfg, $dst_volname);
7004
7005         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7006
7007         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7008     }
7009
7010     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7011     $opts->{format} = $format if $format;
7012
7013     if (defined($src_bitmap)) {
7014         $opts->{sync} = 'incremental';
7015         $opts->{bitmap} = $src_bitmap;
7016         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7017     }
7018
7019     if (defined($bwlimit)) {
7020         $opts->{speed} = $bwlimit * 1024;
7021         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7022     } else {
7023         print "drive mirror is starting for drive-$drive\n";
7024     }
7025
7026     # if a job already runs for this device we get an error, catch it for cleanup
7027     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7028     if (my $err = $@) {
7029         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7030         warn "$@\n" if $@;
7031         die "mirroring error: $err\n";
7032     }
7033
7034     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7035 }
7036
7037 # $completion can be either
7038 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7039 # 'cancel': wait until all jobs are ready, block-job-cancel them
7040 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7041 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7042 sub qemu_drive_mirror_monitor {
7043     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7044
7045     $completion //= 'complete';
7046     $op //= "mirror";
7047
7048     eval {
7049         my $err_complete = 0;
7050
7051         my $starttime = time ();
7052         while (1) {
7053             die "block job ('$op') timed out\n" if $err_complete > 300;
7054
7055             my $stats = mon_cmd($vmid, "query-block-jobs");
7056             my $ctime = time();
7057
7058             my $running_jobs = {};
7059             for my $stat (@$stats) {
7060                 next if $stat->{type} ne $op;
7061                 $running_jobs->{$stat->{device}} = $stat;
7062             }
7063
7064             my $readycounter = 0;
7065
7066             for my $job_id (sort keys %$jobs) {
7067                 my $job = $running_jobs->{$job_id};
7068
7069                 my $vanished = !defined($job);
7070                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7071                 if($complete || ($vanished && $completion eq 'auto')) {
7072                     print "$job_id: $op-job finished\n";
7073                     delete $jobs->{$job_id};
7074                     next;
7075                 }
7076
7077                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7078
7079                 my $busy = $job->{busy};
7080                 my $ready = $job->{ready};
7081                 if (my $total = $job->{len}) {
7082                     my $transferred = $job->{offset} || 0;
7083                     my $remaining = $total - $transferred;
7084                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7085
7086                     my $duration = $ctime - $starttime;
7087                     my $total_h = render_bytes($total, 1);
7088                     my $transferred_h = render_bytes($transferred, 1);
7089
7090                     my $status = sprintf(
7091                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7092                         render_duration($duration),
7093                     );
7094
7095                     if ($ready) {
7096                         if ($busy) {
7097                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7098                         } else {
7099                             $status .= ", ready";
7100                         }
7101                     }
7102                     print "$job_id: $status\n";
7103                 }
7104
7105                 $readycounter++ if $job->{ready};
7106             }
7107
7108             last if scalar(keys %$jobs) == 0;
7109
7110             if ($readycounter == scalar(keys %$jobs)) {
7111                 print "all '$op' jobs are ready\n";
7112
7113                 # do the complete later (or has already been done)
7114                 last if $completion eq 'skip' || $completion eq 'auto';
7115
7116                 if ($vmiddst && $vmiddst != $vmid) {
7117                     my $agent_running = $qga && qga_check_running($vmid);
7118                     if ($agent_running) {
7119                         print "freeze filesystem\n";
7120                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7121                     } else {
7122                         print "suspend vm\n";
7123                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7124                     }
7125
7126                     # if we clone a disk for a new target vm, we don't switch the disk
7127                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7128
7129                     if ($agent_running) {
7130                         print "unfreeze filesystem\n";
7131                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7132                     } else {
7133                         print "resume vm\n";
7134                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7135                     }
7136
7137                     last;
7138                 } else {
7139
7140                     for my $job_id (sort keys %$jobs) {
7141                         # try to switch the disk if source and destination are on the same guest
7142                         print "$job_id: Completing block job_id...\n";
7143
7144                         my $op;
7145                         if ($completion eq 'complete') {
7146                             $op = 'block-job-complete';
7147                         } elsif ($completion eq 'cancel') {
7148                             $op = 'block-job-cancel';
7149                         } else {
7150                             die "invalid completion value: $completion\n";
7151                         }
7152                         eval { mon_cmd($vmid, $op, device => $job_id) };
7153                         if ($@ =~ m/cannot be completed/) {
7154                             print "$job_id: block job cannot be completed, trying again.\n";
7155                             $err_complete++;
7156                         }else {
7157                             print "$job_id: Completed successfully.\n";
7158                             $jobs->{$job_id}->{complete} = 1;
7159                         }
7160                     }
7161                 }
7162             }
7163             sleep 1;
7164         }
7165     };
7166     my $err = $@;
7167
7168     if ($err) {
7169         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7170         die "block job ($op) error: $err";
7171     }
7172 }
7173
7174 sub qemu_blockjobs_cancel {
7175     my ($vmid, $jobs) = @_;
7176
7177     foreach my $job (keys %$jobs) {
7178         print "$job: Cancelling block job\n";
7179         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7180         $jobs->{$job}->{cancel} = 1;
7181     }
7182
7183     while (1) {
7184         my $stats = mon_cmd($vmid, "query-block-jobs");
7185
7186         my $running_jobs = {};
7187         foreach my $stat (@$stats) {
7188             $running_jobs->{$stat->{device}} = $stat;
7189         }
7190
7191         foreach my $job (keys %$jobs) {
7192
7193             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7194                 print "$job: Done.\n";
7195                 delete $jobs->{$job};
7196             }
7197         }
7198
7199         last if scalar(keys %$jobs) == 0;
7200
7201         sleep 1;
7202     }
7203 }
7204
7205 sub clone_disk {
7206     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7207         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7208
7209     my $newvolid;
7210
7211     if (!$full) {
7212         print "create linked clone of drive $drivename ($drive->{file})\n";
7213         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7214         push @$newvollist, $newvolid;
7215     } else {
7216
7217         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7218         $storeid = $storage if $storage;
7219
7220         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7221
7222         print "create full clone of drive $drivename ($drive->{file})\n";
7223         my $name = undef;
7224         my $size = undef;
7225         if (drive_is_cloudinit($drive)) {
7226             $name = "vm-$newvmid-cloudinit";
7227             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7228             if ($scfg->{path}) {
7229                 $name .= ".$dst_format";
7230             }
7231             $snapname = undef;
7232             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7233         } elsif ($drivename eq 'efidisk0') {
7234             $size = get_efivars_size($conf);
7235         } else {
7236             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7237         }
7238         $newvolid = PVE::Storage::vdisk_alloc(
7239             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7240         );
7241         push @$newvollist, $newvolid;
7242
7243         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7244
7245         if (drive_is_cloudinit($drive)) {
7246             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7247             # if this is the case, we have to complete any block-jobs still there from
7248             # previous drive-mirrors
7249             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7250                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7251             }
7252             goto no_data_clone;
7253         }
7254
7255         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7256         if (!$running || $snapname) {
7257             # TODO: handle bwlimits
7258             if ($drivename eq 'efidisk0') {
7259                 # the relevant data on the efidisk may be smaller than the source
7260                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7261                 # that is given by the OVMF_VARS.fd
7262                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7263                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7264
7265                 # better for Ceph if block size is not too small, see bug #3324
7266                 my $bs = 1024*1024;
7267
7268                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7269                     "if=$src_path", "of=$dst_path"]);
7270             } else {
7271                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7272             }
7273         } else {
7274
7275             my $kvmver = get_running_qemu_version ($vmid);
7276             if (!min_version($kvmver, 2, 7)) {
7277                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7278                     if $drive->{iothread};
7279             }
7280
7281             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7282                 $completion, $qga, $bwlimit);
7283         }
7284     }
7285
7286 no_data_clone:
7287     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7288
7289     my $disk = $drive;
7290     $disk->{format} = undef;
7291     $disk->{file} = $newvolid;
7292     $disk->{size} = $size if defined($size);
7293
7294     return $disk;
7295 }
7296
7297 sub get_running_qemu_version {
7298     my ($vmid) = @_;
7299     my $res = mon_cmd($vmid, "query-version");
7300     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7301 }
7302
7303 sub qemu_use_old_bios_files {
7304     my ($machine_type) = @_;
7305
7306     return if !$machine_type;
7307
7308     my $use_old_bios_files = undef;
7309
7310     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7311         $machine_type = $1;
7312         $use_old_bios_files = 1;
7313     } else {
7314         my $version = extract_version($machine_type, kvm_user_version());
7315         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7316         # load new efi bios files on migration. So this hack is required to allow
7317         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7318         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7319         $use_old_bios_files = !min_version($version, 2, 4);
7320     }
7321
7322     return ($use_old_bios_files, $machine_type);
7323 }
7324
7325 sub get_efivars_size {
7326     my ($conf) = @_;
7327     my $arch = get_vm_arch($conf);
7328     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7329     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7330     return -s $ovmf_vars;
7331 }
7332
7333 sub update_efidisk_size {
7334     my ($conf) = @_;
7335
7336     return if !defined($conf->{efidisk0});
7337
7338     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7339     $disk->{size} = get_efivars_size($conf);
7340     $conf->{efidisk0} = print_drive($disk);
7341
7342     return;
7343 }
7344
7345 sub create_efidisk($$$$$) {
7346     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7347
7348     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7349     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7350
7351     my $vars_size_b = -s $ovmf_vars;
7352     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7353     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7354     PVE::Storage::activate_volumes($storecfg, [$volid]);
7355
7356     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7357     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7358
7359     return ($volid, $size/1024);
7360 }
7361
7362 sub vm_iothreads_list {
7363     my ($vmid) = @_;
7364
7365     my $res = mon_cmd($vmid, 'query-iothreads');
7366
7367     my $iothreads = {};
7368     foreach my $iothread (@$res) {
7369         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7370     }
7371
7372     return $iothreads;
7373 }
7374
7375 sub scsihw_infos {
7376     my ($conf, $drive) = @_;
7377
7378     my $maxdev = 0;
7379
7380     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7381         $maxdev = 7;
7382     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7383         $maxdev = 1;
7384     } else {
7385         $maxdev = 256;
7386     }
7387
7388     my $controller = int($drive->{index} / $maxdev);
7389     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7390         ? "virtioscsi"
7391         : "scsihw";
7392
7393     return ($maxdev, $controller, $controller_prefix);
7394 }
7395
7396 sub windows_version {
7397     my ($ostype) = @_;
7398
7399     return 0 if !$ostype;
7400
7401     my $winversion = 0;
7402
7403     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7404         $winversion = 5;
7405     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7406         $winversion = 6;
7407     } elsif ($ostype =~ m/^win(\d+)$/) {
7408         $winversion = $1;
7409     }
7410
7411     return $winversion;
7412 }
7413
7414 sub resolve_dst_disk_format {
7415         my ($storecfg, $storeid, $src_volname, $format) = @_;
7416         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7417
7418         if (!$format) {
7419             # if no target format is specified, use the source disk format as hint
7420             if ($src_volname) {
7421                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7422                 $format = qemu_img_format($scfg, $src_volname);
7423             } else {
7424                 return $defFormat;
7425             }
7426         }
7427
7428         # test if requested format is supported - else use default
7429         my $supported = grep { $_ eq $format } @$validFormats;
7430         $format = $defFormat if !$supported;
7431         return $format;
7432 }
7433
7434 # NOTE: if this logic changes, please update docs & possibly gui logic
7435 sub find_vmstate_storage {
7436     my ($conf, $storecfg) = @_;
7437
7438     # first, return storage from conf if set
7439     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7440
7441     my ($target, $shared, $local);
7442
7443     foreach_storage_used_by_vm($conf, sub {
7444         my ($sid) = @_;
7445         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7446         my $dst = $scfg->{shared} ? \$shared : \$local;
7447         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7448     });
7449
7450     # second, use shared storage where VM has at least one disk
7451     # third, use local storage where VM has at least one disk
7452     # fall back to local storage
7453     $target = $shared // $local // 'local';
7454
7455     return $target;
7456 }
7457
7458 sub generate_uuid {
7459     my ($uuid, $uuid_str);
7460     UUID::generate($uuid);
7461     UUID::unparse($uuid, $uuid_str);
7462     return $uuid_str;
7463 }
7464
7465 sub generate_smbios1_uuid {
7466     return "uuid=".generate_uuid();
7467 }
7468
7469 sub nbd_stop {
7470     my ($vmid) = @_;
7471
7472     mon_cmd($vmid, 'nbd-server-stop');
7473 }
7474
7475 sub create_reboot_request {
7476     my ($vmid) = @_;
7477     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7478         or die "failed to create reboot trigger file: $!\n";
7479     close($fh);
7480 }
7481
7482 sub clear_reboot_request {
7483     my ($vmid) = @_;
7484     my $path = "/run/qemu-server/$vmid.reboot";
7485     my $res = 0;
7486
7487     $res = unlink($path);
7488     die "could not remove reboot request for $vmid: $!"
7489         if !$res && $! != POSIX::ENOENT;
7490
7491     return $res;
7492 }
7493
7494 sub bootorder_from_legacy {
7495     my ($conf, $bootcfg) = @_;
7496
7497     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7498     my $bootindex_hash = {};
7499     my $i = 1;
7500     foreach my $o (split(//, $boot)) {
7501         $bootindex_hash->{$o} = $i*100;
7502         $i++;
7503     }
7504
7505     my $bootorder = {};
7506
7507     PVE::QemuConfig->foreach_volume($conf, sub {
7508         my ($ds, $drive) = @_;
7509
7510         if (drive_is_cdrom ($drive, 1)) {
7511             if ($bootindex_hash->{d}) {
7512                 $bootorder->{$ds} = $bootindex_hash->{d};
7513                 $bootindex_hash->{d} += 1;
7514             }
7515         } elsif ($bootindex_hash->{c}) {
7516             $bootorder->{$ds} = $bootindex_hash->{c}
7517                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7518             $bootindex_hash->{c} += 1;
7519         }
7520     });
7521
7522     if ($bootindex_hash->{n}) {
7523         for (my $i = 0; $i < $MAX_NETS; $i++) {
7524             my $netname = "net$i";
7525             next if !$conf->{$netname};
7526             $bootorder->{$netname} = $bootindex_hash->{n};
7527             $bootindex_hash->{n} += 1;
7528         }
7529     }
7530
7531     return $bootorder;
7532 }
7533
7534 # Generate default device list for 'boot: order=' property. Matches legacy
7535 # default boot order, but with explicit device names. This is important, since
7536 # the fallback for when neither 'order' nor the old format is specified relies
7537 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7538 sub get_default_bootdevices {
7539     my ($conf) = @_;
7540
7541     my @ret = ();
7542
7543     # harddisk
7544     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7545     push @ret, $first if $first;
7546
7547     # cdrom
7548     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7549     push @ret, $first if $first;
7550
7551     # network
7552     for (my $i = 0; $i < $MAX_NETS; $i++) {
7553         my $netname = "net$i";
7554         next if !$conf->{$netname};
7555         push @ret, $netname;
7556         last;
7557     }
7558
7559     return \@ret;
7560 }
7561
7562 sub device_bootorder {
7563     my ($conf) = @_;
7564
7565     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7566
7567     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7568
7569     my $bootorder = {};
7570     if (!defined($boot) || $boot->{legacy}) {
7571         $bootorder = bootorder_from_legacy($conf, $boot);
7572     } elsif ($boot->{order}) {
7573         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7574         for my $dev (PVE::Tools::split_list($boot->{order})) {
7575             $bootorder->{$dev} = $i++;
7576         }
7577     }
7578
7579     return $bootorder;
7580 }
7581
7582 sub register_qmeventd_handle {
7583     my ($vmid) = @_;
7584
7585     my $fh;
7586     my $peer = "/var/run/qmeventd.sock";
7587     my $count = 0;
7588
7589     for (;;) {
7590         $count++;
7591         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7592         last if $fh;
7593         if ($! != EINTR && $! != EAGAIN) {
7594             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7595         }
7596         if ($count > 4) {
7597             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7598               . "after $count retries\n";
7599         }
7600         usleep(25000);
7601     }
7602
7603     # send handshake to mark VM as backing up
7604     print $fh to_json({vzdump => {vmid => "$vmid"}});
7605
7606     # return handle to be closed later when inhibit is no longer required
7607     return $fh;
7608 }
7609
7610 # bash completion helper
7611
7612 sub complete_backup_archives {
7613     my ($cmdname, $pname, $cvalue) = @_;
7614
7615     my $cfg = PVE::Storage::config();
7616
7617     my $storeid;
7618
7619     if ($cvalue =~ m/^([^:]+):/) {
7620         $storeid = $1;
7621     }
7622
7623     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7624
7625     my $res = [];
7626     foreach my $id (keys %$data) {
7627         foreach my $item (@{$data->{$id}}) {
7628             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7629             push @$res, $item->{volid} if defined($item->{volid});
7630         }
7631     }
7632
7633     return $res;
7634 }
7635
7636 my $complete_vmid_full = sub {
7637     my ($running) = @_;
7638
7639     my $idlist = vmstatus();
7640
7641     my $res = [];
7642
7643     foreach my $id (keys %$idlist) {
7644         my $d = $idlist->{$id};
7645         if (defined($running)) {
7646             next if $d->{template};
7647             next if $running && $d->{status} ne 'running';
7648             next if !$running && $d->{status} eq 'running';
7649         }
7650         push @$res, $id;
7651
7652     }
7653     return $res;
7654 };
7655
7656 sub complete_vmid {
7657     return &$complete_vmid_full();
7658 }
7659
7660 sub complete_vmid_stopped {
7661     return &$complete_vmid_full(0);
7662 }
7663
7664 sub complete_vmid_running {
7665     return &$complete_vmid_full(1);
7666 }
7667
7668 sub complete_storage {
7669
7670     my $cfg = PVE::Storage::config();
7671     my $ids = $cfg->{ids};
7672
7673     my $res = [];
7674     foreach my $sid (keys %$ids) {
7675         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7676         next if !$ids->{$sid}->{content}->{images};
7677         push @$res, $sid;
7678     }
7679
7680     return $res;
7681 }
7682
7683 sub complete_migration_storage {
7684     my ($cmd, $param, $current_value, $all_args) = @_;
7685
7686     my $targetnode = @$all_args[1];
7687
7688     my $cfg = PVE::Storage::config();
7689     my $ids = $cfg->{ids};
7690
7691     my $res = [];
7692     foreach my $sid (keys %$ids) {
7693         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7694         next if !$ids->{$sid}->{content}->{images};
7695         push @$res, $sid;
7696     }
7697
7698     return $res;
7699 }
7700
7701 sub vm_is_paused {
7702     my ($vmid) = @_;
7703     my $qmpstatus = eval {
7704         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7705         mon_cmd($vmid, "query-status");
7706     };
7707     warn "$@\n" if $@;
7708     return $qmpstatus && $qmpstatus->{status} eq "paused";
7709 }
7710
7711 1;