PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016/2019
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 968 cloud-init 19.4 or newer.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 971 dhcp on IPv4.
 972 EODESCR
 973 };
 974 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 975
 976 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 977     $confdesc->{"net$i"} = $netdesc;
 978     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 979 }
 980
 981 foreach my $key (keys %$confdesc_cloudinit) {
 982     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 983 }
 984
 985 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 986 sub verify_volume_id_or_qm_path {
 987     my ($volid, $noerr) = @_;
 988
 989     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 990         return $volid;
 991     }
 992
 993     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 994     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 995     if ($@) {
 996         return if $noerr;
 997         die $@;
 998     }
 999     return $volid;
1000 }
1001
1002 my $usb_fmt = {
1003     host => {
1004         default_key => 1,
1005         type => 'string', format => 'pve-qm-usb-device',
1006         format_description => 'HOSTUSBDEVICE|spice',
1007         description => <<EODESCR,
1008 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1009
1010  'bus-port(.port)*' (decimal numbers) or
1011  'vendor_id:product_id' (hexadeciaml numbers) or
1012  'spice'
1013
1014 You can use the 'lsusb -t' command to list existing usb devices.
1015
1016 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1017 machines - use with special care.
1018
1019 The value 'spice' can be used to add a usb redirection devices for spice.
1020 EODESCR
1021     },
1022     usb3 => {
1023         optional => 1,
1024         type => 'boolean',
1025         description => "Specifies whether if given host option is a USB3 device or port.",
1026         default => 0,
1027     },
1028 };
1029
1030 my $usbdesc = {
1031     optional => 1,
1032     type => 'string', format => $usb_fmt,
1033     description => "Configure an USB device (n is 0 to 4).",
1034 };
1035 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1036
1037 my $serialdesc = {
1038         optional => 1,
1039         type => 'string',
1040         pattern => '(/dev/.+|socket)',
1041         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1042         verbose_description =>  <<EODESCR,
1043 Create a serial device inside the VM (n is 0 to 3), and pass through a
1044 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1045 host side (use 'qm terminal' to open a terminal connection).
1046
1047 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1048 use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053
1054 my $paralleldesc= {
1055         optional => 1,
1056         type => 'string',
1057         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1058         description =>  "Map host parallel devices (n is 0 to 2).",
1059         verbose_description =>  <<EODESCR,
1060 Map host parallel devices (n is 0 to 2).
1061
1062 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1063 machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068
1069 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1070     $confdesc->{"parallel$i"} = $paralleldesc;
1071 }
1072
1073 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1074     $confdesc->{"serial$i"} = $serialdesc;
1075 }
1076
1077 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1078     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1079 }
1080
1081 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1082     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1083 }
1084
1085 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1086     $confdesc->{"usb$i"} = $usbdesc;
1087 }
1088
1089 my $boot_fmt = {
1090     legacy => {
1091         optional => 1,
1092         default_key => 1,
1093         type => 'string',
1094         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1095                      . " Deprecated, use 'order=' instead.",
1096         pattern => '[acdn]{1,4}',
1097         format_description => "[acdn]{1,4}",
1098
1099         # note: this is also the fallback if boot: is not given at all
1100         default => 'cdn',
1101     },
1102     order => {
1103         optional => 1,
1104         type => 'string',
1105         format => 'pve-qm-bootdev-list',
1106         format_description => "device[;device...]",
1107         description => <<EODESC,
1108 The guest will attempt to boot from devices in the order they appear here.
1109
1110 Disks, optical drives and passed-through storage USB devices will be directly
1111 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1112 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1113
1114 Note that only devices in this list will be marked as bootable and thus loaded
1115 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1116 (e.g. software-raid), you need to specify all of them here.
1117
1118 Overrides the deprecated 'legacy=[acdn]*' value when given.
1119 EODESC
1120     },
1121 };
1122 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1123
1124 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1125 sub verify_bootdev {
1126     my ($dev, $noerr) = @_;
1127
1128     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1129
1130     my $check = sub {
1131         my ($base) = @_;
1132         return 0 if $dev !~ m/^$base\d+$/;
1133         return 0 if !$confdesc->{$dev};
1134         return 1;
1135     };
1136
1137     return $dev if $check->("net");
1138     return $dev if $check->("usb");
1139     return $dev if $check->("hostpci");
1140
1141     return if $noerr;
1142     die "invalid boot device '$dev'\n";
1143 }
1144
1145 sub print_bootorder {
1146     my ($devs) = @_;
1147     return "" if !@$devs;
1148     my $data = { order => join(';', @$devs) };
1149     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1150 }
1151
1152 my $kvm_api_version = 0;
1153
1154 sub kvm_version {
1155     return $kvm_api_version if $kvm_api_version;
1156
1157     open my $fh, '<', '/dev/kvm' or return;
1158
1159     # 0xae00 => KVM_GET_API_VERSION
1160     $kvm_api_version = ioctl($fh, 0xae00, 0);
1161     close($fh);
1162
1163     return $kvm_api_version;
1164 }
1165
1166 my $kvm_user_version = {};
1167 my $kvm_mtime = {};
1168
1169 sub kvm_user_version {
1170     my ($binary) = @_;
1171
1172     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1173     my $st = stat($binary);
1174
1175     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1176     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1177         $cachedmtime == $st->mtime;
1178
1179     $kvm_user_version->{$binary} = 'unknown';
1180     $kvm_mtime->{$binary} = $st->mtime;
1181
1182     my $code = sub {
1183         my $line = shift;
1184         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1185             $kvm_user_version->{$binary} = $2;
1186         }
1187     };
1188
1189     eval { run_command([$binary, '--version'], outfunc => $code); };
1190     warn $@ if $@;
1191
1192     return $kvm_user_version->{$binary};
1193
1194 }
1195 my sub extract_version {
1196     my ($machine_type, $version) = @_;
1197     $version = kvm_user_version() if !defined($version);
1198     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1199 }
1200
1201 sub kernel_has_vhost_net {
1202     return -c '/dev/vhost-net';
1203 }
1204
1205 sub option_exists {
1206     my $key = shift;
1207     return defined($confdesc->{$key});
1208 }
1209
1210 my $cdrom_path;
1211 sub get_cdrom_path {
1212
1213     return  $cdrom_path if $cdrom_path;
1214
1215     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1216     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1217     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1218 }
1219
1220 sub get_iso_path {
1221     my ($storecfg, $vmid, $cdrom) = @_;
1222
1223     if ($cdrom eq 'cdrom') {
1224         return get_cdrom_path();
1225     } elsif ($cdrom eq 'none') {
1226         return '';
1227     } elsif ($cdrom =~ m|^/|) {
1228         return $cdrom;
1229     } else {
1230         return PVE::Storage::path($storecfg, $cdrom);
1231     }
1232 }
1233
1234 # try to convert old style file names to volume IDs
1235 sub filename_to_volume_id {
1236     my ($vmid, $file, $media) = @_;
1237
1238      if (!($file eq 'none' || $file eq 'cdrom' ||
1239           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1240
1241         return if $file =~ m|/|;
1242
1243         if ($media && $media eq 'cdrom') {
1244             $file = "local:iso/$file";
1245         } else {
1246             $file = "local:$vmid/$file";
1247         }
1248     }
1249
1250     return $file;
1251 }
1252
1253 sub verify_media_type {
1254     my ($opt, $vtype, $media) = @_;
1255
1256     return if !$media;
1257
1258     my $etype;
1259     if ($media eq 'disk') {
1260         $etype = 'images';
1261     } elsif ($media eq 'cdrom') {
1262         $etype = 'iso';
1263     } else {
1264         die "internal error";
1265     }
1266
1267     return if ($vtype eq $etype);
1268
1269     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1270 }
1271
1272 sub cleanup_drive_path {
1273     my ($opt, $storecfg, $drive) = @_;
1274
1275     # try to convert filesystem paths to volume IDs
1276
1277     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1278         ($drive->{file} !~ m|^/dev/.+|) &&
1279         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1280         ($drive->{file} !~ m/^\d+$/)) {
1281         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1282         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1283             if !$vtype;
1284         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1285         verify_media_type($opt, $vtype, $drive->{media});
1286         $drive->{file} = $volid;
1287     }
1288
1289     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1290 }
1291
1292 sub parse_hotplug_features {
1293     my ($data) = @_;
1294
1295     my $res = {};
1296
1297     return $res if $data eq '0';
1298
1299     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1300
1301     foreach my $feature (PVE::Tools::split_list($data)) {
1302         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1303             $res->{$1} = 1;
1304         } else {
1305             die "invalid hotplug feature '$feature'\n";
1306         }
1307     }
1308     return $res;
1309 }
1310
1311 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1312 sub pve_verify_hotplug_features {
1313     my ($value, $noerr) = @_;
1314
1315     return $value if parse_hotplug_features($value);
1316
1317     return if $noerr;
1318
1319     die "unable to parse hotplug option\n";
1320 }
1321
1322 sub scsi_inquiry {
1323     my($fh, $noerr) = @_;
1324
1325     my $SG_IO = 0x2285;
1326     my $SG_GET_VERSION_NUM = 0x2282;
1327
1328     my $versionbuf = "\x00" x 8;
1329     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1330     if (!$ret) {
1331         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1332         return;
1333     }
1334     my $version = unpack("I", $versionbuf);
1335     if ($version < 30000) {
1336         die "scsi generic interface too old\n"  if !$noerr;
1337         return;
1338     }
1339
1340     my $buf = "\x00" x 36;
1341     my $sensebuf = "\x00" x 8;
1342     my $cmd = pack("C x3 C x1", 0x12, 36);
1343
1344     # see /usr/include/scsi/sg.h
1345     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1346
1347     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1348                       length($sensebuf), 0, length($buf), $buf,
1349                       $cmd, $sensebuf, 6000);
1350
1351     $ret = ioctl($fh, $SG_IO, $packet);
1352     if (!$ret) {
1353         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1354         return;
1355     }
1356
1357     my @res = unpack($sg_io_hdr_t, $packet);
1358     if ($res[17] || $res[18]) {
1359         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1360         return;
1361     }
1362
1363     my $res = {};
1364     (my $byte0, my $byte1, $res->{vendor},
1365      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1366
1367     $res->{removable} = $byte1 & 128 ? 1 : 0;
1368     $res->{type} = $byte0 & 31;
1369
1370     return $res;
1371 }
1372
1373 sub path_is_scsi {
1374     my ($path) = @_;
1375
1376     my $fh = IO::File->new("+<$path") || return;
1377     my $res = scsi_inquiry($fh, 1);
1378     close($fh);
1379
1380     return $res;
1381 }
1382
1383 sub print_tabletdevice_full {
1384     my ($conf, $arch) = @_;
1385
1386     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1387
1388     # we use uhci for old VMs because tablet driver was buggy in older qemu
1389     my $usbbus;
1390     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1391         $usbbus = 'ehci';
1392     } else {
1393         $usbbus = 'uhci';
1394     }
1395
1396     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1397 }
1398
1399 sub print_keyboarddevice_full {
1400     my ($conf, $arch, $machine) = @_;
1401
1402     return if $arch ne 'aarch64';
1403
1404     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1405 }
1406
1407 sub print_drivedevice_full {
1408     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1409
1410     my $device = '';
1411     my $maxdev = 0;
1412
1413     my $drive_id = "$drive->{interface}$drive->{index}";
1414     if ($drive->{interface} eq 'virtio') {
1415         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1416         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1417         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1418     } elsif ($drive->{interface} eq 'scsi') {
1419
1420         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1421         my $unit = $drive->{index} % $maxdev;
1422         my $devicetype = 'hd';
1423         my $path = '';
1424         if (drive_is_cdrom($drive)) {
1425             $devicetype = 'cd';
1426         } else {
1427             if ($drive->{file} =~ m|^/|) {
1428                 $path = $drive->{file};
1429                 if (my $info = path_is_scsi($path)) {
1430                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1431                         $devicetype = 'block';
1432                     } elsif ($info->{type} == 1) { # tape
1433                         $devicetype = 'generic';
1434                     }
1435                 }
1436             } else {
1437                  $path = PVE::Storage::path($storecfg, $drive->{file});
1438             }
1439
1440             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1441             my $version = extract_version($machine_type, kvm_user_version());
1442             if ($path =~ m/^iscsi\:\/\// &&
1443                !min_version($version, 4, 1)) {
1444                 $devicetype = 'generic';
1445             }
1446         }
1447
1448         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1449             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1450         } else {
1451             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1452                 .",lun=$drive->{index}";
1453         }
1454         $device .= ",drive=drive-$drive_id,id=$drive_id";
1455
1456         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1457             $device .= ",rotation_rate=1";
1458         }
1459         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1460
1461     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1462         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1463         my $controller = int($drive->{index} / $maxdev);
1464         my $unit = $drive->{index} % $maxdev;
1465         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1466
1467         $device = "ide-$devicetype";
1468         if ($drive->{interface} eq 'ide') {
1469             $device .= ",bus=ide.$controller,unit=$unit";
1470         } else {
1471             $device .= ",bus=ahci$controller.$unit";
1472         }
1473         $device .= ",drive=drive-$drive_id,id=$drive_id";
1474
1475         if ($devicetype eq 'hd') {
1476             if (my $model = $drive->{model}) {
1477                 $model = URI::Escape::uri_unescape($model);
1478                 $device .= ",model=$model";
1479             }
1480             if ($drive->{ssd}) {
1481                 $device .= ",rotation_rate=1";
1482             }
1483         }
1484         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1485     } elsif ($drive->{interface} eq 'usb') {
1486         die "implement me";
1487         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1488     } else {
1489         die "unsupported interface type";
1490     }
1491
1492     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1493
1494     if (my $serial = $drive->{serial}) {
1495         $serial = URI::Escape::uri_unescape($serial);
1496         $device .= ",serial=$serial";
1497     }
1498
1499
1500     return $device;
1501 }
1502
1503 sub get_initiator_name {
1504     my $initiator;
1505
1506     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1507     while (defined(my $line = <$fh>)) {
1508         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1509         $initiator = $1;
1510         last;
1511     }
1512     $fh->close();
1513
1514     return $initiator;
1515 }
1516
1517 sub print_drive_commandline_full {
1518     my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1519
1520     my $path;
1521     my $volid = $drive->{file};
1522     my $format = $drive->{format};
1523
1524     if (drive_is_cdrom($drive)) {
1525         $path = get_iso_path($storecfg, $vmid, $volid);
1526         die "cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1527     } else {
1528         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1529         if ($storeid) {
1530             $path = PVE::Storage::path($storecfg, $volid);
1531             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1532             $format //= qemu_img_format($scfg, $volname);
1533         } else {
1534             $path = $volid;
1535             $format //= "raw";
1536         }
1537    }
1538
1539    my $is_rbd = $path =~ m/^rbd:/;
1540
1541     my $opts = '';
1542     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1543     foreach my $o (@qemu_drive_options) {
1544         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1545     }
1546
1547     # snapshot only accepts on|off
1548     if (defined($drive->{snapshot})) {
1549         my $v = $drive->{snapshot} ? 'on' : 'off';
1550         $opts .= ",snapshot=$v";
1551     }
1552
1553     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1554         my ($dir, $qmpname) = @$type;
1555         if (my $v = $drive->{"mbps$dir"}) {
1556             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1557         }
1558         if (my $v = $drive->{"mbps${dir}_max"}) {
1559             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1560         }
1561         if (my $v = $drive->{"bps${dir}_max_length"}) {
1562             $opts .= ",throttling.bps$qmpname-max-length=$v";
1563         }
1564         if (my $v = $drive->{"iops${dir}"}) {
1565             $opts .= ",throttling.iops$qmpname=$v";
1566         }
1567         if (my $v = $drive->{"iops${dir}_max"}) {
1568             $opts .= ",throttling.iops$qmpname-max=$v";
1569         }
1570         if (my $v = $drive->{"iops${dir}_max_length"}) {
1571             $opts .= ",throttling.iops$qmpname-max-length=$v";
1572         }
1573     }
1574
1575     if ($pbs_name) {
1576         $format = "rbd" if $is_rbd;
1577         die "PBS backing requires a drive with known format\n" if !$format;
1578         $opts .= ",format=alloc-track,file.driver=$format";
1579     } elsif ($format) {
1580         $opts .= ",format=$format";
1581     }
1582
1583     my $cache_direct = 0;
1584
1585     if (my $cache = $drive->{cache}) {
1586         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1587     } elsif (!drive_is_cdrom($drive)) {
1588         $opts .= ",cache=none";
1589         $cache_direct = 1;
1590     }
1591
1592     # aio native works only with O_DIRECT
1593     if (!$drive->{aio}) {
1594         if($cache_direct) {
1595             $opts .= ",aio=native";
1596         } else {
1597             $opts .= ",aio=threads";
1598         }
1599     }
1600
1601     if (!drive_is_cdrom($drive)) {
1602         my $detectzeroes;
1603         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1604             $detectzeroes = 'off';
1605         } elsif ($drive->{discard}) {
1606             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1607         } else {
1608             # This used to be our default with discard not being specified:
1609             $detectzeroes = 'on';
1610         }
1611
1612         # note: 'detect-zeroes' works per blockdev and we want it to persist
1613         # after the alloc-track is removed, so put it on 'file' directly
1614         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1615         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1616     }
1617
1618     if ($pbs_name) {
1619         $opts .= ",backing=$pbs_name";
1620         $opts .= ",auto-remove=on";
1621     }
1622
1623     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1624     my $file_param = "file";
1625     if ($pbs_name) {
1626         # non-rbd drivers require the underlying file to be a seperate block
1627         # node, so add a second .file indirection
1628         $file_param .= ".file" if !$is_rbd;
1629         $file_param .= ".filename";
1630     }
1631     my $pathinfo = $path ? "$file_param=$path," : '';
1632
1633     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1634 }
1635
1636 sub print_pbs_blockdev {
1637     my ($pbs_conf, $pbs_name) = @_;
1638     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1639     $blockdev .= ",repository=$pbs_conf->{repository}";
1640     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1641     $blockdev .= ",archive=$pbs_conf->{archive}";
1642     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1643     return $blockdev;
1644 }
1645
1646 sub print_netdevice_full {
1647     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1648
1649     my $device = $net->{model};
1650     if ($net->{model} eq 'virtio') {
1651          $device = 'virtio-net-pci';
1652      };
1653
1654     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1655     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1656     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1657         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1658         # and out of each queue plus one config interrupt and control vector queue
1659         my $vectors = $net->{queues} * 2 + 2;
1660         $tmpstr .= ",vectors=$vectors,mq=on";
1661     }
1662     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1663
1664     if (my $mtu = $net->{mtu}) {
1665         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1666             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1667             if ($mtu == 1) {
1668                  $mtu = $bridge_mtu;
1669             } elsif ($mtu < 576) {
1670                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1671             } elsif ($mtu > $bridge_mtu) {
1672                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1673             }
1674             $tmpstr .= ",host_mtu=$mtu";
1675         } else {
1676             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1677         }
1678     }
1679
1680     if ($use_old_bios_files) {
1681         my $romfile;
1682         if ($device eq 'virtio-net-pci') {
1683             $romfile = 'pxe-virtio.rom';
1684         } elsif ($device eq 'e1000') {
1685             $romfile = 'pxe-e1000.rom';
1686         } elsif ($device eq 'ne2k') {
1687             $romfile = 'pxe-ne2k_pci.rom';
1688         } elsif ($device eq 'pcnet') {
1689             $romfile = 'pxe-pcnet.rom';
1690         } elsif ($device eq 'rtl8139') {
1691             $romfile = 'pxe-rtl8139.rom';
1692         }
1693         $tmpstr .= ",romfile=$romfile" if $romfile;
1694     }
1695
1696     return $tmpstr;
1697 }
1698
1699 sub print_netdev_full {
1700     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1701
1702     my $i = '';
1703     if ($netid =~ m/^net(\d+)$/) {
1704         $i = int($1);
1705     }
1706
1707     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1708
1709     my $ifname = "tap${vmid}i$i";
1710
1711     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1712     die "interface name '$ifname' is too long (max 15 character)\n"
1713         if length($ifname) >= 16;
1714
1715     my $vhostparam = '';
1716     if (is_native($arch)) {
1717         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1718     }
1719
1720     my $vmname = $conf->{name} || "vm$vmid";
1721
1722     my $netdev = "";
1723     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1724
1725     if ($net->{bridge}) {
1726         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1727             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1728     } else {
1729         $netdev = "type=user,id=$netid,hostname=$vmname";
1730     }
1731
1732     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1733
1734     return $netdev;
1735 }
1736
1737 my $vga_map = {
1738     'cirrus' => 'cirrus-vga',
1739     'std' => 'VGA',
1740     'vmware' => 'vmware-svga',
1741     'virtio' => 'virtio-vga',
1742 };
1743
1744 sub print_vga_device {
1745     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1746
1747     my $type = $vga_map->{$vga->{type}};
1748     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1749         $type = 'virtio-gpu';
1750     }
1751     my $vgamem_mb = $vga->{memory};
1752
1753     my $max_outputs = '';
1754     if ($qxlnum) {
1755         $type = $id ? 'qxl' : 'qxl-vga';
1756
1757         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1758             # set max outputs so linux can have up to 4 qxl displays with one device
1759             if (min_version($machine_version, 4, 1)) {
1760                 $max_outputs = ",max_outputs=4";
1761             }
1762         }
1763     }
1764
1765     die "no devicetype for $vga->{type}\n" if !$type;
1766
1767     my $memory = "";
1768     if ($vgamem_mb) {
1769         if ($vga->{type} eq 'virtio') {
1770             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1771             $memory = ",max_hostmem=$bytes";
1772         } elsif ($qxlnum) {
1773             # from https://www.spice-space.org/multiple-monitors.html
1774             $memory = ",vgamem_mb=$vga->{memory}";
1775             my $ram = $vgamem_mb * 4;
1776             my $vram = $vgamem_mb * 2;
1777             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1778         } else {
1779             $memory = ",vgamem_mb=$vga->{memory}";
1780         }
1781     } elsif ($qxlnum && $id) {
1782         $memory = ",ram_size=67108864,vram_size=33554432";
1783     }
1784
1785     my $edidoff = "";
1786     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1787         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1788     }
1789
1790     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1791     my $vgaid = "vga" . ($id // '');
1792     my $pciaddr;
1793
1794     if ($q35 && $vgaid eq 'vga') {
1795         # the first display uses pcie.0 bus on q35 machines
1796         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1797     } else {
1798         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1799     }
1800
1801     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1802 }
1803
1804 sub parse_number_sets {
1805     my ($set) = @_;
1806     my $res = [];
1807     foreach my $part (split(/;/, $set)) {
1808         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1809             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1810             push @$res, [ $1, $2 ];
1811         } else {
1812             die "invalid range: $part\n";
1813         }
1814     }
1815     return $res;
1816 }
1817
1818 sub parse_numa {
1819     my ($data) = @_;
1820
1821     my $res = parse_property_string($numa_fmt, $data);
1822     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1823     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1824     return $res;
1825 }
1826
1827 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1828 sub parse_net {
1829     my ($data) = @_;
1830
1831     my $res = eval { parse_property_string($net_fmt, $data) };
1832     if ($@) {
1833         warn $@;
1834         return;
1835     }
1836     if (!defined($res->{macaddr})) {
1837         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1838         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1839     }
1840     return $res;
1841 }
1842
1843 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1844 sub parse_ipconfig {
1845     my ($data) = @_;
1846
1847     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1848     if ($@) {
1849         warn $@;
1850         return;
1851     }
1852
1853     if ($res->{gw} && !$res->{ip}) {
1854         warn 'gateway specified without specifying an IP address';
1855         return;
1856     }
1857     if ($res->{gw6} && !$res->{ip6}) {
1858         warn 'IPv6 gateway specified without specifying an IPv6 address';
1859         return;
1860     }
1861     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1862         warn 'gateway specified together with DHCP';
1863         return;
1864     }
1865     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1866         # gw6 + auto/dhcp
1867         warn "IPv6 gateway specified together with $res->{ip6} address";
1868         return;
1869     }
1870
1871     if (!$res->{ip} && !$res->{ip6}) {
1872         return { ip => 'dhcp', ip6 => 'dhcp' };
1873     }
1874
1875     return $res;
1876 }
1877
1878 sub print_net {
1879     my $net = shift;
1880
1881     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1882 }
1883
1884 sub add_random_macs {
1885     my ($settings) = @_;
1886
1887     foreach my $opt (keys %$settings) {
1888         next if $opt !~ m/^net(\d+)$/;
1889         my $net = parse_net($settings->{$opt});
1890         next if !$net;
1891         $settings->{$opt} = print_net($net);
1892     }
1893 }
1894
1895 sub vm_is_volid_owner {
1896     my ($storecfg, $vmid, $volid) = @_;
1897
1898     if ($volid !~  m|^/|) {
1899         my ($path, $owner);
1900         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1901         if ($owner && ($owner == $vmid)) {
1902             return 1;
1903         }
1904     }
1905
1906     return;
1907 }
1908
1909 sub vmconfig_register_unused_drive {
1910     my ($storecfg, $vmid, $conf, $drive) = @_;
1911
1912     if (drive_is_cloudinit($drive)) {
1913         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1914         warn $@ if $@;
1915     } elsif (!drive_is_cdrom($drive)) {
1916         my $volid = $drive->{file};
1917         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1918             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1919         }
1920     }
1921 }
1922
1923 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1924 my $smbios1_fmt = {
1925     uuid => {
1926         type => 'string',
1927         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1928         format_description => 'UUID',
1929         description => "Set SMBIOS1 UUID.",
1930         optional => 1,
1931     },
1932     version => {
1933         type => 'string',
1934         pattern => '[A-Za-z0-9+\/]+={0,2}',
1935         format_description => 'Base64 encoded string',
1936         description => "Set SMBIOS1 version.",
1937         optional => 1,
1938     },
1939     serial => {
1940         type => 'string',
1941         pattern => '[A-Za-z0-9+\/]+={0,2}',
1942         format_description => 'Base64 encoded string',
1943         description => "Set SMBIOS1 serial number.",
1944         optional => 1,
1945     },
1946     manufacturer => {
1947         type => 'string',
1948         pattern => '[A-Za-z0-9+\/]+={0,2}',
1949         format_description => 'Base64 encoded string',
1950         description => "Set SMBIOS1 manufacturer.",
1951         optional => 1,
1952     },
1953     product => {
1954         type => 'string',
1955         pattern => '[A-Za-z0-9+\/]+={0,2}',
1956         format_description => 'Base64 encoded string',
1957         description => "Set SMBIOS1 product ID.",
1958         optional => 1,
1959     },
1960     sku => {
1961         type => 'string',
1962         pattern => '[A-Za-z0-9+\/]+={0,2}',
1963         format_description => 'Base64 encoded string',
1964         description => "Set SMBIOS1 SKU string.",
1965         optional => 1,
1966     },
1967     family => {
1968         type => 'string',
1969         pattern => '[A-Za-z0-9+\/]+={0,2}',
1970         format_description => 'Base64 encoded string',
1971         description => "Set SMBIOS1 family string.",
1972         optional => 1,
1973     },
1974     base64 => {
1975         type => 'boolean',
1976         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1977         optional => 1,
1978     },
1979 };
1980
1981 sub parse_smbios1 {
1982     my ($data) = @_;
1983
1984     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1985     warn $@ if $@;
1986     return $res;
1987 }
1988
1989 sub print_smbios1 {
1990     my ($smbios1) = @_;
1991     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1992 }
1993
1994 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1995
1996 sub parse_watchdog {
1997     my ($value) = @_;
1998
1999     return if !$value;
2000
2001     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2002     warn $@ if $@;
2003     return $res;
2004 }
2005
2006 sub parse_guest_agent {
2007     my ($conf) = @_;
2008
2009     return {} if !defined($conf->{agent});
2010
2011     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2012     warn $@ if $@;
2013
2014     # if the agent is disabled ignore the other potentially set properties
2015     return {} if !$res->{enabled};
2016     return $res;
2017 }
2018
2019 sub get_qga_key {
2020     my ($conf, $key) = @_;
2021     return undef if !defined($conf->{agent});
2022
2023     my $agent = parse_guest_agent($conf);
2024     return $agent->{$key};
2025 }
2026
2027 sub parse_vga {
2028     my ($value) = @_;
2029
2030     return {} if !$value;
2031     my $res = eval { parse_property_string($vga_fmt, $value) };
2032     warn $@ if $@;
2033     return $res;
2034 }
2035
2036 sub parse_rng {
2037     my ($value) = @_;
2038
2039     return if !$value;
2040
2041     my $res = eval { parse_property_string($rng_fmt, $value) };
2042     warn $@ if $@;
2043     return $res;
2044 }
2045
2046 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2047 sub verify_usb_device {
2048     my ($value, $noerr) = @_;
2049
2050     return $value if parse_usb_device($value);
2051
2052     return if $noerr;
2053
2054     die "unable to parse usb device\n";
2055 }
2056
2057 # add JSON properties for create and set function
2058 sub json_config_properties {
2059     my $prop = shift;
2060
2061     foreach my $opt (keys %$confdesc) {
2062         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2063             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2064         $prop->{$opt} = $confdesc->{$opt};
2065     }
2066
2067     return $prop;
2068 }
2069
2070 # return copy of $confdesc_cloudinit to generate documentation
2071 sub cloudinit_config_properties {
2072
2073     return dclone($confdesc_cloudinit);
2074 }
2075
2076 sub check_type {
2077     my ($key, $value) = @_;
2078
2079     die "unknown setting '$key'\n" if !$confdesc->{$key};
2080
2081     my $type = $confdesc->{$key}->{type};
2082
2083     if (!defined($value)) {
2084         die "got undefined value\n";
2085     }
2086
2087     if ($value =~ m/[\n\r]/) {
2088         die "property contains a line feed\n";
2089     }
2090
2091     if ($type eq 'boolean') {
2092         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2093         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2094         die "type check ('boolean') failed - got '$value'\n";
2095     } elsif ($type eq 'integer') {
2096         return int($1) if $value =~ m/^(\d+)$/;
2097         die "type check ('integer') failed - got '$value'\n";
2098     } elsif ($type eq 'number') {
2099         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2100         die "type check ('number') failed - got '$value'\n";
2101     } elsif ($type eq 'string') {
2102         if (my $fmt = $confdesc->{$key}->{format}) {
2103             PVE::JSONSchema::check_format($fmt, $value);
2104             return $value;
2105         }
2106         $value =~ s/^\"(.*)\"$/$1/;
2107         return $value;
2108     } else {
2109         die "internal error"
2110     }
2111 }
2112
2113 sub destroy_vm {
2114     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2115
2116     my $conf = PVE::QemuConfig->load_config($vmid);
2117
2118     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2119
2120     if ($conf->{template}) {
2121         # check if any base image is still used by a linked clone
2122         PVE::QemuConfig->foreach_volume($conf, sub {
2123                 my ($ds, $drive) = @_;
2124                 return if drive_is_cdrom($drive);
2125
2126                 my $volid = $drive->{file};
2127                 return if !$volid || $volid =~ m|^/|;
2128
2129                 die "base volume '$volid' is still in use by linked cloned\n"
2130                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2131
2132         });
2133     }
2134
2135     # only remove disks owned by this VM (referenced in the config)
2136     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2137         my ($ds, $drive) = @_;
2138         return if drive_is_cdrom($drive, 1);
2139
2140         my $volid = $drive->{file};
2141         return if !$volid || $volid =~ m|^/|;
2142
2143         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2144         return if !$path || !$owner || ($owner != $vmid);
2145
2146         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2147         warn "Could not remove disk '$volid', check manually: $@" if $@;
2148     });
2149
2150     if ($purge_unreferenced) { # also remove unreferenced disk
2151         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2152         PVE::Storage::foreach_volid($vmdisks, sub {
2153             my ($volid, $sid, $volname, $d) = @_;
2154             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2155             warn $@ if $@;
2156         });
2157     }
2158
2159     if (defined $replacement_conf) {
2160         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2161     } else {
2162         PVE::QemuConfig->destroy_config($vmid);
2163     }
2164 }
2165
2166 sub parse_vm_config {
2167     my ($filename, $raw) = @_;
2168
2169     return if !defined($raw);
2170
2171     my $res = {
2172         digest => Digest::SHA::sha1_hex($raw),
2173         snapshots => {},
2174         pending => {},
2175     };
2176
2177     $filename =~ m|/qemu-server/(\d+)\.conf$|
2178         || die "got strange filename '$filename'";
2179
2180     my $vmid = $1;
2181
2182     my $conf = $res;
2183     my $descr;
2184     my $section = '';
2185
2186     my @lines = split(/\n/, $raw);
2187     foreach my $line (@lines) {
2188         next if $line =~ m/^\s*$/;
2189
2190         if ($line =~ m/^\[PENDING\]\s*$/i) {
2191             $section = 'pending';
2192             if (defined($descr)) {
2193                 $descr =~ s/\s+$//;
2194                 $conf->{description} = $descr;
2195             }
2196             $descr = undef;
2197             $conf = $res->{$section} = {};
2198             next;
2199
2200         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2201             $section = $1;
2202             if (defined($descr)) {
2203                 $descr =~ s/\s+$//;
2204                 $conf->{description} = $descr;
2205             }
2206             $descr = undef;
2207             $conf = $res->{snapshots}->{$section} = {};
2208             next;
2209         }
2210
2211         if ($line =~ m/^\#(.*)\s*$/) {
2212             $descr = '' if !defined($descr);
2213             $descr .= PVE::Tools::decode_text($1) . "\n";
2214             next;
2215         }
2216
2217         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2218             $descr = '' if !defined($descr);
2219             $descr .= PVE::Tools::decode_text($2);
2220         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2221             $conf->{snapstate} = $1;
2222         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2223             my $key = $1;
2224             my $value = $2;
2225             $conf->{$key} = $value;
2226         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2227             my $value = $1;
2228             if ($section eq 'pending') {
2229                 $conf->{delete} = $value; # we parse this later
2230             } else {
2231                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2232             }
2233         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2234             my $key = $1;
2235             my $value = $2;
2236             eval { $value = check_type($key, $value); };
2237             if ($@) {
2238                 warn "vm $vmid - unable to parse value of '$key' - $@";
2239             } else {
2240                 $key = 'ide2' if $key eq 'cdrom';
2241                 my $fmt = $confdesc->{$key}->{format};
2242                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2243                     my $v = parse_drive($key, $value);
2244                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2245                         $v->{file} = $volid;
2246                         $value = print_drive($v);
2247                     } else {
2248                         warn "vm $vmid - unable to parse value of '$key'\n";
2249                         next;
2250                     }
2251                 }
2252
2253                 $conf->{$key} = $value;
2254             }
2255         } else {
2256             warn "vm $vmid - unable to parse config: $line\n";
2257         }
2258     }
2259
2260     if (defined($descr)) {
2261         $descr =~ s/\s+$//;
2262         $conf->{description} = $descr;
2263     }
2264     delete $res->{snapstate}; # just to be sure
2265
2266     return $res;
2267 }
2268
2269 sub write_vm_config {
2270     my ($filename, $conf) = @_;
2271
2272     delete $conf->{snapstate}; # just to be sure
2273
2274     if ($conf->{cdrom}) {
2275         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2276         $conf->{ide2} = $conf->{cdrom};
2277         delete $conf->{cdrom};
2278     }
2279
2280     # we do not use 'smp' any longer
2281     if ($conf->{sockets}) {
2282         delete $conf->{smp};
2283     } elsif ($conf->{smp}) {
2284         $conf->{sockets} = $conf->{smp};
2285         delete $conf->{cores};
2286         delete $conf->{smp};
2287     }
2288
2289     my $used_volids = {};
2290
2291     my $cleanup_config = sub {
2292         my ($cref, $pending, $snapname) = @_;
2293
2294         foreach my $key (keys %$cref) {
2295             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2296                 $key eq 'snapstate' || $key eq 'pending';
2297             my $value = $cref->{$key};
2298             if ($key eq 'delete') {
2299                 die "propertry 'delete' is only allowed in [PENDING]\n"
2300                     if !$pending;
2301                 # fixme: check syntax?
2302                 next;
2303             }
2304             eval { $value = check_type($key, $value); };
2305             die "unable to parse value of '$key' - $@" if $@;
2306
2307             $cref->{$key} = $value;
2308
2309             if (!$snapname && is_valid_drivename($key)) {
2310                 my $drive = parse_drive($key, $value);
2311                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2312             }
2313         }
2314     };
2315
2316     &$cleanup_config($conf);
2317
2318     &$cleanup_config($conf->{pending}, 1);
2319
2320     foreach my $snapname (keys %{$conf->{snapshots}}) {
2321         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2322         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2323     }
2324
2325     # remove 'unusedX' settings if we re-add a volume
2326     foreach my $key (keys %$conf) {
2327         my $value = $conf->{$key};
2328         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2329             delete $conf->{$key};
2330         }
2331     }
2332
2333     my $generate_raw_config = sub {
2334         my ($conf, $pending) = @_;
2335
2336         my $raw = '';
2337
2338         # add description as comment to top of file
2339         if (defined(my $descr = $conf->{description})) {
2340             if ($descr) {
2341                 foreach my $cl (split(/\n/, $descr)) {
2342                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2343                 }
2344             } else {
2345                 $raw .= "#\n" if $pending;
2346             }
2347         }
2348
2349         foreach my $key (sort keys %$conf) {
2350             next if $key =~ /^(digest|description|pending|snapshots)$/;
2351             $raw .= "$key: $conf->{$key}\n";
2352         }
2353         return $raw;
2354     };
2355
2356     my $raw = &$generate_raw_config($conf);
2357
2358     if (scalar(keys %{$conf->{pending}})){
2359         $raw .= "\n[PENDING]\n";
2360         $raw .= &$generate_raw_config($conf->{pending}, 1);
2361     }
2362
2363     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2364         $raw .= "\n[$snapname]\n";
2365         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2366     }
2367
2368     return $raw;
2369 }
2370
2371 sub load_defaults {
2372
2373     my $res = {};
2374
2375     # we use static defaults from our JSON schema configuration
2376     foreach my $key (keys %$confdesc) {
2377         if (defined(my $default = $confdesc->{$key}->{default})) {
2378             $res->{$key} = $default;
2379         }
2380     }
2381
2382     return $res;
2383 }
2384
2385 sub config_list {
2386     my $vmlist = PVE::Cluster::get_vmlist();
2387     my $res = {};
2388     return $res if !$vmlist || !$vmlist->{ids};
2389     my $ids = $vmlist->{ids};
2390     my $nodename = nodename();
2391
2392     foreach my $vmid (keys %$ids) {
2393         my $d = $ids->{$vmid};
2394         next if !$d->{node} || $d->{node} ne $nodename;
2395         next if !$d->{type} || $d->{type} ne 'qemu';
2396         $res->{$vmid}->{exists} = 1;
2397     }
2398     return $res;
2399 }
2400
2401 # test if VM uses local resources (to prevent migration)
2402 sub check_local_resources {
2403     my ($conf, $noerr) = @_;
2404
2405     my @loc_res = ();
2406
2407     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2408     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2409
2410     push @loc_res, "ivshmem" if $conf->{ivshmem};
2411
2412     foreach my $k (keys %$conf) {
2413         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2414         # sockets are safe: they will recreated be on the target side post-migrate
2415         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2416         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2417     }
2418
2419     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2420
2421     return \@loc_res;
2422 }
2423
2424 # check if used storages are available on all nodes (use by migrate)
2425 sub check_storage_availability {
2426     my ($storecfg, $conf, $node) = @_;
2427
2428     PVE::QemuConfig->foreach_volume($conf, sub {
2429         my ($ds, $drive) = @_;
2430
2431         my $volid = $drive->{file};
2432         return if !$volid;
2433
2434         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2435         return if !$sid;
2436
2437         # check if storage is available on both nodes
2438         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2439         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2440    });
2441 }
2442
2443 # list nodes where all VM images are available (used by has_feature API)
2444 sub shared_nodes {
2445     my ($conf, $storecfg) = @_;
2446
2447     my $nodelist = PVE::Cluster::get_nodelist();
2448     my $nodehash = { map { $_ => 1 } @$nodelist };
2449     my $nodename = nodename();
2450
2451     PVE::QemuConfig->foreach_volume($conf, sub {
2452         my ($ds, $drive) = @_;
2453
2454         my $volid = $drive->{file};
2455         return if !$volid;
2456
2457         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2458         if ($storeid) {
2459             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2460             if ($scfg->{disable}) {
2461                 $nodehash = {};
2462             } elsif (my $avail = $scfg->{nodes}) {
2463                 foreach my $node (keys %$nodehash) {
2464                     delete $nodehash->{$node} if !$avail->{$node};
2465                 }
2466             } elsif (!$scfg->{shared}) {
2467                 foreach my $node (keys %$nodehash) {
2468                     delete $nodehash->{$node} if $node ne $nodename
2469                 }
2470             }
2471         }
2472     });
2473
2474     return $nodehash
2475 }
2476
2477 sub check_local_storage_availability {
2478     my ($conf, $storecfg) = @_;
2479
2480     my $nodelist = PVE::Cluster::get_nodelist();
2481     my $nodehash = { map { $_ => {} } @$nodelist };
2482
2483     PVE::QemuConfig->foreach_volume($conf, sub {
2484         my ($ds, $drive) = @_;
2485
2486         my $volid = $drive->{file};
2487         return if !$volid;
2488
2489         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2490         if ($storeid) {
2491             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2492
2493             if ($scfg->{disable}) {
2494                 foreach my $node (keys %$nodehash) {
2495                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2496                 }
2497             } elsif (my $avail = $scfg->{nodes}) {
2498                 foreach my $node (keys %$nodehash) {
2499                     if (!$avail->{$node}) {
2500                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2501                     }
2502                 }
2503             }
2504         }
2505     });
2506
2507     foreach my $node (values %$nodehash) {
2508         if (my $unavail = $node->{unavailable_storages}) {
2509             $node->{unavailable_storages} = [ sort keys %$unavail ];
2510         }
2511     }
2512
2513     return $nodehash
2514 }
2515
2516 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2517 sub check_running {
2518     my ($vmid, $nocheck, $node) = @_;
2519
2520     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2521     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2522 }
2523
2524 sub vzlist {
2525
2526     my $vzlist = config_list();
2527
2528     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2529
2530     while (defined(my $de = $fd->read)) {
2531         next if $de !~ m/^(\d+)\.pid$/;
2532         my $vmid = $1;
2533         next if !defined($vzlist->{$vmid});
2534         if (my $pid = check_running($vmid)) {
2535             $vzlist->{$vmid}->{pid} = $pid;
2536         }
2537     }
2538
2539     return $vzlist;
2540 }
2541
2542 our $vmstatus_return_properties = {
2543     vmid => get_standard_option('pve-vmid'),
2544     status => {
2545         description => "Qemu process status.",
2546         type => 'string',
2547         enum => ['stopped', 'running'],
2548     },
2549     maxmem => {
2550         description => "Maximum memory in bytes.",
2551         type => 'integer',
2552         optional => 1,
2553         renderer => 'bytes',
2554     },
2555     maxdisk => {
2556         description => "Root disk size in bytes.",
2557         type => 'integer',
2558         optional => 1,
2559         renderer => 'bytes',
2560     },
2561     name => {
2562         description => "VM name.",
2563         type => 'string',
2564         optional => 1,
2565     },
2566     qmpstatus => {
2567         description => "Qemu QMP agent status.",
2568         type => 'string',
2569         optional => 1,
2570     },
2571     pid => {
2572         description => "PID of running qemu process.",
2573         type => 'integer',
2574         optional => 1,
2575     },
2576     uptime => {
2577         description => "Uptime.",
2578         type => 'integer',
2579         optional => 1,
2580         renderer => 'duration',
2581     },
2582     cpus => {
2583         description => "Maximum usable CPUs.",
2584         type => 'number',
2585         optional => 1,
2586     },
2587     lock => {
2588         description => "The current config lock, if any.",
2589         type => 'string',
2590         optional => 1,
2591     },
2592     tags => {
2593         description  => "The current configured tags, if any",
2594         type => 'string',
2595         optional => 1,
2596     },
2597     'running-machine' => {
2598         description => "The currently running machine type (if running).",
2599         type => 'string',
2600         optional => 1,
2601     },
2602     'running-qemu' => {
2603         description => "The currently running QEMU version (if running).",
2604         type => 'string',
2605         optional => 1,
2606     },
2607 };
2608
2609 my $last_proc_pid_stat;
2610
2611 # get VM status information
2612 # This must be fast and should not block ($full == false)
2613 # We only query KVM using QMP if $full == true (this can be slow)
2614 sub vmstatus {
2615     my ($opt_vmid, $full) = @_;
2616
2617     my $res = {};
2618
2619     my $storecfg = PVE::Storage::config();
2620
2621     my $list = vzlist();
2622     my $defaults = load_defaults();
2623
2624     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2625
2626     my $cpucount = $cpuinfo->{cpus} || 1;
2627
2628     foreach my $vmid (keys %$list) {
2629         next if $opt_vmid && ($vmid ne $opt_vmid);
2630
2631         my $conf = PVE::QemuConfig->load_config($vmid);
2632
2633         my $d = { vmid => $vmid };
2634         $d->{pid} = $list->{$vmid}->{pid};
2635
2636         # fixme: better status?
2637         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2638
2639         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2640         if (defined($size)) {
2641             $d->{disk} = 0; # no info available
2642             $d->{maxdisk} = $size;
2643         } else {
2644             $d->{disk} = 0;
2645             $d->{maxdisk} = 0;
2646         }
2647
2648         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2649             * ($conf->{cores} || $defaults->{cores});
2650         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2651         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2652
2653         $d->{name} = $conf->{name} || "VM $vmid";
2654         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2655             : $defaults->{memory}*(1024*1024);
2656
2657         if ($conf->{balloon}) {
2658             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2659             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2660                 : $defaults->{shares};
2661         }
2662
2663         $d->{uptime} = 0;
2664         $d->{cpu} = 0;
2665         $d->{mem} = 0;
2666
2667         $d->{netout} = 0;
2668         $d->{netin} = 0;
2669
2670         $d->{diskread} = 0;
2671         $d->{diskwrite} = 0;
2672
2673         $d->{template} = PVE::QemuConfig->is_template($conf);
2674
2675         $d->{serial} = 1 if conf_has_serial($conf);
2676         $d->{lock} = $conf->{lock} if $conf->{lock};
2677         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2678
2679         $res->{$vmid} = $d;
2680     }
2681
2682     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2683     foreach my $dev (keys %$netdev) {
2684         next if $dev !~ m/^tap([1-9]\d*)i/;
2685         my $vmid = $1;
2686         my $d = $res->{$vmid};
2687         next if !$d;
2688
2689         $d->{netout} += $netdev->{$dev}->{receive};
2690         $d->{netin} += $netdev->{$dev}->{transmit};
2691
2692         if ($full) {
2693             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2694             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2695         }
2696
2697     }
2698
2699     my $ctime = gettimeofday;
2700
2701     foreach my $vmid (keys %$list) {
2702
2703         my $d = $res->{$vmid};
2704         my $pid = $d->{pid};
2705         next if !$pid;
2706
2707         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2708         next if !$pstat; # not running
2709
2710         my $used = $pstat->{utime} + $pstat->{stime};
2711
2712         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2713
2714         if ($pstat->{vsize}) {
2715             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2716         }
2717
2718         my $old = $last_proc_pid_stat->{$pid};
2719         if (!$old) {
2720             $last_proc_pid_stat->{$pid} = {
2721                 time => $ctime,
2722                 used => $used,
2723                 cpu => 0,
2724             };
2725             next;
2726         }
2727
2728         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2729
2730         if ($dtime > 1000) {
2731             my $dutime = $used -  $old->{used};
2732
2733             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2734             $last_proc_pid_stat->{$pid} = {
2735                 time => $ctime,
2736                 used => $used,
2737                 cpu => $d->{cpu},
2738             };
2739         } else {
2740             $d->{cpu} = $old->{cpu};
2741         }
2742     }
2743
2744     return $res if !$full;
2745
2746     my $qmpclient = PVE::QMPClient->new();
2747
2748     my $ballooncb = sub {
2749         my ($vmid, $resp) = @_;
2750
2751         my $info = $resp->{'return'};
2752         return if !$info->{max_mem};
2753
2754         my $d = $res->{$vmid};
2755
2756         # use memory assigned to VM
2757         $d->{maxmem} = $info->{max_mem};
2758         $d->{balloon} = $info->{actual};
2759
2760         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2761             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2762             $d->{freemem} = $info->{free_mem};
2763         }
2764
2765         $d->{ballooninfo} = $info;
2766     };
2767
2768     my $blockstatscb = sub {
2769         my ($vmid, $resp) = @_;
2770         my $data = $resp->{'return'} || [];
2771         my $totalrdbytes = 0;
2772         my $totalwrbytes = 0;
2773
2774         for my $blockstat (@$data) {
2775             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2776             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2777
2778             $blockstat->{device} =~ s/drive-//;
2779             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2780         }
2781         $res->{$vmid}->{diskread} = $totalrdbytes;
2782         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2783     };
2784
2785     my $machinecb = sub {
2786         my ($vmid, $resp) = @_;
2787         my $data = $resp->{'return'} || [];
2788
2789         $res->{$vmid}->{'running-machine'} =
2790             PVE::QemuServer::Machine::current_from_query_machines($data);
2791     };
2792
2793     my $versioncb = sub {
2794         my ($vmid, $resp) = @_;
2795         my $data = $resp->{'return'} // {};
2796         my $version = 'unknown';
2797
2798         if (my $v = $data->{qemu}) {
2799             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2800         }
2801
2802         $res->{$vmid}->{'running-qemu'} = $version;
2803     };
2804
2805     my $statuscb = sub {
2806         my ($vmid, $resp) = @_;
2807
2808         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2809         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2810         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2811         # this fails if ballon driver is not loaded, so this must be
2812         # the last commnand (following command are aborted if this fails).
2813         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2814
2815         my $status = 'unknown';
2816         if (!defined($status = $resp->{'return'}->{status})) {
2817             warn "unable to get VM status\n";
2818             return;
2819         }
2820
2821         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2822     };
2823
2824     foreach my $vmid (keys %$list) {
2825         next if $opt_vmid && ($vmid ne $opt_vmid);
2826         next if !$res->{$vmid}->{pid}; # not running
2827         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2828     }
2829
2830     $qmpclient->queue_execute(undef, 2);
2831
2832     foreach my $vmid (keys %$list) {
2833         next if $opt_vmid && ($vmid ne $opt_vmid);
2834         next if !$res->{$vmid}->{pid}; #not running
2835
2836         # we can't use the $qmpclient since it might have already aborted on
2837         # 'query-balloon', but this might also fail for older versions...
2838         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2839         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2840     }
2841
2842     foreach my $vmid (keys %$list) {
2843         next if $opt_vmid && ($vmid ne $opt_vmid);
2844         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2845     }
2846
2847     return $res;
2848 }
2849
2850 sub conf_has_serial {
2851     my ($conf) = @_;
2852
2853     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2854         if ($conf->{"serial$i"}) {
2855             return 1;
2856         }
2857     }
2858
2859     return 0;
2860 }
2861
2862 sub conf_has_audio {
2863     my ($conf, $id) = @_;
2864
2865     $id //= 0;
2866     my $audio = $conf->{"audio$id"};
2867     return if !defined($audio);
2868
2869     my $audioproperties = parse_property_string($audio_fmt, $audio);
2870     my $audiodriver = $audioproperties->{driver} // 'spice';
2871
2872     return {
2873         dev => $audioproperties->{device},
2874         dev_id => "audiodev$id",
2875         backend => $audiodriver,
2876         backend_id => "$audiodriver-backend${id}",
2877     };
2878 }
2879
2880 sub audio_devs {
2881     my ($audio, $audiopciaddr, $machine_version) = @_;
2882
2883     my $devs = [];
2884
2885     my $id = $audio->{dev_id};
2886     my $audiodev = "";
2887     if (min_version($machine_version, 4, 2)) {
2888         $audiodev = ",audiodev=$audio->{backend_id}";
2889     }
2890
2891     if ($audio->{dev} eq 'AC97') {
2892         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2893     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2894         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2895         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2896         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2897     } else {
2898         die "unkown audio device '$audio->{dev}', implement me!";
2899     }
2900
2901     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2902
2903     return $devs;
2904 }
2905
2906 sub vga_conf_has_spice {
2907     my ($vga) = @_;
2908
2909     my $vgaconf = parse_vga($vga);
2910     my $vgatype = $vgaconf->{type};
2911     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2912
2913     return $1 || 1;
2914 }
2915
2916 sub is_native($) {
2917     my ($arch) = @_;
2918     return get_host_arch() eq $arch;
2919 }
2920
2921 sub get_vm_arch {
2922     my ($conf) = @_;
2923     return $conf->{arch} // get_host_arch();
2924 }
2925
2926 my $default_machines = {
2927     x86_64 => 'pc',
2928     aarch64 => 'virt',
2929 };
2930
2931 sub get_installed_machine_version {
2932     my ($kvmversion) = @_;
2933     $kvmversion = kvm_user_version() if !defined($kvmversion);
2934     $kvmversion =~ m/^(\d+\.\d+)/;
2935     return $1;
2936 }
2937
2938 sub windows_get_pinned_machine_version {
2939     my ($machine, $base_version, $kvmversion) = @_;
2940
2941     my $pin_version = $base_version;
2942     if (!defined($base_version) ||
2943         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2944     ) {
2945         $pin_version = get_installed_machine_version($kvmversion);
2946     }
2947     if (!$machine || $machine eq 'pc') {
2948         $machine = "pc-i440fx-$pin_version";
2949     } elsif ($machine eq 'q35') {
2950         $machine = "pc-q35-$pin_version";
2951     } elsif ($machine eq 'virt') {
2952         $machine = "virt-$pin_version";
2953     } else {
2954         warn "unknown machine type '$machine', not touching that!\n";
2955     }
2956
2957     return $machine;
2958 }
2959
2960 sub get_vm_machine {
2961     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2962
2963     my $machine = $forcemachine || $conf->{machine};
2964
2965     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2966         $kvmversion //= kvm_user_version();
2967         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2968         # layout which confuses windows quite a bit and may result in various regressions..
2969         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2970         if (windows_version($conf->{ostype})) {
2971             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2972         }
2973         $arch //= 'x86_64';
2974         $machine ||= $default_machines->{$arch};
2975         if ($add_pve_version) {
2976             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2977             $machine .= "+pve$pvever";
2978         }
2979     }
2980
2981     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2982         # for version-pinned machines that do not include a pve-version (e.g.
2983         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2984         $machine .= '+pve0';
2985     }
2986
2987     return $machine;
2988 }
2989
2990 sub get_ovmf_files($) {
2991     my ($arch) = @_;
2992
2993     my $ovmf = $OVMF->{$arch}
2994         or die "no OVMF images known for architecture '$arch'\n";
2995
2996     return @$ovmf;
2997 }
2998
2999 my $Arch2Qemu = {
3000     aarch64 => '/usr/bin/qemu-system-aarch64',
3001     x86_64 => '/usr/bin/qemu-system-x86_64',
3002 };
3003 sub get_command_for_arch($) {
3004     my ($arch) = @_;
3005     return '/usr/bin/kvm' if is_native($arch);
3006
3007     my $cmd = $Arch2Qemu->{$arch}
3008         or die "don't know how to emulate architecture '$arch'\n";
3009     return $cmd;
3010 }
3011
3012 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3013 # to use in a QEMU command line (-cpu element), first array_intersect the result
3014 # of query_supported_ with query_understood_. This is necessary because:
3015 #
3016 # a) query_understood_ returns flags the host cannot use and
3017 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3018 #    flags, but CPU settings - with most of them being flags. Those settings
3019 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3020 #
3021 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3022 # expensive. If you need the value returned from this, you can get it much
3023 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3024 # $accel being 'kvm' or 'tcg'.
3025 #
3026 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3027 # changes, automatically populating pmxcfs.
3028 #
3029 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3030 # since kvm and tcg machines support different flags
3031 #
3032 sub query_supported_cpu_flags {
3033     my ($arch) = @_;
3034
3035     $arch //= get_host_arch();
3036     my $default_machine = $default_machines->{$arch};
3037
3038     my $flags = {};
3039
3040     # FIXME: Once this is merged, the code below should work for ARM as well:
3041     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3042     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3043         $arch eq "aarch64";
3044
3045     my $kvm_supported = defined(kvm_version());
3046     my $qemu_cmd = get_command_for_arch($arch);
3047     my $fakevmid = -1;
3048     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3049
3050     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3051     my $query_supported_run_qemu = sub {
3052         my ($kvm) = @_;
3053
3054         my $flags = {};
3055         my $cmd = [
3056             $qemu_cmd,
3057             '-machine', $default_machine,
3058             '-display', 'none',
3059             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3060             '-mon', 'chardev=qmp,mode=control',
3061             '-pidfile', $pidfile,
3062             '-S', '-daemonize'
3063         ];
3064
3065         if (!$kvm) {
3066             push @$cmd, '-accel', 'tcg';
3067         }
3068
3069         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3070         die "QEMU flag querying VM exited with code " . $rc if $rc;
3071
3072         eval {
3073             my $cmd_result = mon_cmd(
3074                 $fakevmid,
3075                 'query-cpu-model-expansion',
3076                 type => 'full',
3077                 model => { name => 'host' }
3078             );
3079
3080             my $props = $cmd_result->{model}->{props};
3081             foreach my $prop (keys %$props) {
3082                 next if $props->{$prop} ne '1';
3083                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3084                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3085                 # We only keep those with underscores, to match /proc/cpuinfo
3086                 $prop =~ s/\.|-/_/g;
3087                 $flags->{$prop} = 1;
3088             }
3089         };
3090         my $err = $@;
3091
3092         # force stop with 10 sec timeout and 'nocheck'
3093         # always stop, even if QMP failed
3094         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3095
3096         die $err if $err;
3097
3098         return [ sort keys %$flags ];
3099     };
3100
3101     # We need to query QEMU twice, since KVM and TCG have different supported flags
3102     PVE::QemuConfig->lock_config($fakevmid, sub {
3103         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3104         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3105
3106         if ($kvm_supported) {
3107             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3108             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3109         }
3110     });
3111
3112     return $flags;
3113 }
3114
3115 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3116 my $understood_cpu_flag_dir = "/usr/share/kvm";
3117 sub query_understood_cpu_flags {
3118     my $arch = get_host_arch();
3119     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3120
3121     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3122         if ! -e $filepath;
3123
3124     my $raw = file_get_contents($filepath);
3125     $raw =~ s/^\s+|\s+$//g;
3126     my @flags = split(/\s+/, $raw);
3127
3128     return \@flags;
3129 }
3130
3131 sub config_to_command {
3132     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3133         $pbs_backing) = @_;
3134
3135     my $cmd = [];
3136     my $globalFlags = [];
3137     my $machineFlags = [];
3138     my $rtcFlags = [];
3139     my $devices = [];
3140     my $pciaddr = '';
3141     my $bridges = {};
3142     my $ostype = $conf->{ostype};
3143     my $winversion = windows_version($ostype);
3144     my $kvm = $conf->{kvm};
3145     my $nodename = nodename();
3146
3147     my $arch = get_vm_arch($conf);
3148     my $kvm_binary = get_command_for_arch($arch);
3149     my $kvmver = kvm_user_version($kvm_binary);
3150
3151     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3152         $kvmver //= "undefined";
3153         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3154     }
3155
3156     my $add_pve_version = min_version($kvmver, 4, 1);
3157
3158     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3159     my $machine_version = extract_version($machine_type, $kvmver);
3160     $kvm //= 1 if is_native($arch);
3161
3162     $machine_version =~ m/(\d+)\.(\d+)/;
3163     my ($machine_major, $machine_minor) = ($1, $2);
3164
3165     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3166         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3167     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3168         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3169             ." please upgrade node '$nodename'\n"
3170     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3171         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3172         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3173             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3174             ." node '$nodename'\n";
3175     }
3176
3177     # if a specific +pve version is required for a feature, use $version_guard
3178     # instead of min_version to allow machines to be run with the minimum
3179     # required version
3180     my $required_pve_version = 0;
3181     my $version_guard = sub {
3182         my ($major, $minor, $pve) = @_;
3183         return 0 if !min_version($machine_version, $major, $minor, $pve);
3184         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3185         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3186         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3187         return 1;
3188     };
3189
3190     if ($kvm && !defined kvm_version()) {
3191         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3192             ." or enable in BIOS.\n";
3193     }
3194
3195     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3196     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3197     my $use_old_bios_files = undef;
3198     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3199
3200     my $cpuunits = defined($conf->{cpuunits}) ?
3201             $conf->{cpuunits} : $defaults->{cpuunits};
3202
3203     push @$cmd, $kvm_binary;
3204
3205     push @$cmd, '-id', $vmid;
3206
3207     my $vmname = $conf->{name} || "vm$vmid";
3208
3209     push @$cmd, '-name', $vmname;
3210
3211     push @$cmd, '-no-shutdown';
3212
3213     my $use_virtio = 0;
3214
3215     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3216     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3217     push @$cmd, '-mon', "chardev=qmp,mode=control";
3218
3219     if (min_version($machine_version, 2, 12)) {
3220         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3221         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3222     }
3223
3224     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3225
3226     push @$cmd, '-daemonize';
3227
3228     if ($conf->{smbios1}) {
3229         my $smbios_conf = parse_smbios1($conf->{smbios1});
3230         if ($smbios_conf->{base64}) {
3231             # Do not pass base64 flag to qemu
3232             delete $smbios_conf->{base64};
3233             my $smbios_string = "";
3234             foreach my $key (keys %$smbios_conf) {
3235                 my $value;
3236                 if ($key eq "uuid") {
3237                     $value = $smbios_conf->{uuid}
3238                 } else {
3239                     $value = decode_base64($smbios_conf->{$key});
3240                 }
3241                 # qemu accepts any binary data, only commas need escaping by double comma
3242                 $value =~ s/,/,,/g;
3243                 $smbios_string .= "," . $key . "=" . $value if $value;
3244             }
3245             push @$cmd, '-smbios', "type=1" . $smbios_string;
3246         } else {
3247             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3248         }
3249     }
3250
3251     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3252         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3253         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3254
3255         my ($path, $format);
3256         if (my $efidisk = $conf->{efidisk0}) {
3257             my $d = parse_drive('efidisk0', $efidisk);
3258             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3259             $format = $d->{format};
3260             if ($storeid) {
3261                 $path = PVE::Storage::path($storecfg, $d->{file});
3262                 if (!defined($format)) {
3263                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3264                     $format = qemu_img_format($scfg, $volname);
3265                 }
3266             } else {
3267                 $path = $d->{file};
3268                 die "efidisk format must be specified\n"
3269                     if !defined($format);
3270             }
3271         } else {
3272             warn "no efidisk configured! Using temporary efivars disk.\n";
3273             $path = "/tmp/$vmid-ovmf.fd";
3274             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3275             $format = 'raw';
3276         }
3277
3278         my $size_str = "";
3279
3280         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3281             $size_str = ",size=" . (-s $ovmf_vars);
3282         }
3283
3284         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3285         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3286     }
3287
3288     # load q35 config
3289     if ($q35) {
3290         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3291         if (min_version($machine_version, 4, 0)) {
3292             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3293         } else {
3294             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3295         }
3296     }
3297
3298     if ($conf->{vmgenid}) {
3299         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3300     }
3301
3302     # add usb controllers
3303     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3304         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3305     push @$devices, @usbcontrollers if @usbcontrollers;
3306     my $vga = parse_vga($conf->{vga});
3307
3308     my $qxlnum = vga_conf_has_spice($conf->{vga});
3309     $vga->{type} = 'qxl' if $qxlnum;
3310
3311     if (!$vga->{type}) {
3312         if ($arch eq 'aarch64') {
3313             $vga->{type} = 'virtio';
3314         } elsif (min_version($machine_version, 2, 9)) {
3315             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3316         } else {
3317             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3318         }
3319     }
3320
3321     # enable absolute mouse coordinates (needed by vnc)
3322     my $tablet;
3323     if (defined($conf->{tablet})) {
3324         $tablet = $conf->{tablet};
3325     } else {
3326         $tablet = $defaults->{tablet};
3327         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3328         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3329     }
3330
3331     if ($tablet) {
3332         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3333         my $kbd = print_keyboarddevice_full($conf, $arch);
3334         push @$devices, '-device', $kbd if defined($kbd);
3335     }
3336
3337     my $bootorder = device_bootorder($conf);
3338
3339     # host pci device passthrough
3340     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3341         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3342
3343     # usb devices
3344     my $usb_dev_features = {};
3345     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3346
3347     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3348         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3349     push @$devices, @usbdevices if @usbdevices;
3350
3351     # serial devices
3352     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3353         if (my $path = $conf->{"serial$i"}) {
3354             if ($path eq 'socket') {
3355                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3356                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3357                 # On aarch64, serial0 is the UART device. Qemu only allows
3358                 # connecting UART devices via the '-serial' command line, as
3359                 # the device has a fixed slot on the hardware...
3360                 if ($arch eq 'aarch64' && $i == 0) {
3361                     push @$devices, '-serial', "chardev:serial$i";
3362                 } else {
3363                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3364                 }
3365             } else {
3366                 die "no such serial device\n" if ! -c $path;
3367                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3368                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3369             }
3370         }
3371     }
3372
3373     # parallel devices
3374     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3375         if (my $path = $conf->{"parallel$i"}) {
3376             die "no such parallel device\n" if ! -c $path;
3377             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3378             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3379             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3380         }
3381     }
3382
3383     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3384         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3385         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3386         push @$devices, @$audio_devs;
3387     }
3388
3389     my $sockets = 1;
3390     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3391     $sockets = $conf->{sockets} if  $conf->{sockets};
3392
3393     my $cores = $conf->{cores} || 1;
3394
3395     my $maxcpus = $sockets * $cores;
3396
3397     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3398
3399     my $allowed_vcpus = $cpuinfo->{cpus};
3400
3401     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3402         if ($allowed_vcpus < $maxcpus);
3403
3404     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3405
3406         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3407         for (my $i = 2; $i <= $vcpus; $i++)  {
3408             my $cpustr = print_cpu_device($conf,$i);
3409             push @$cmd, '-device', $cpustr;
3410         }
3411
3412     } else {
3413
3414         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3415     }
3416     push @$cmd, '-nodefaults';
3417
3418     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3419
3420     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3421
3422     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3423
3424     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3425         push @$devices, '-device', print_vga_device(
3426             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3427         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3428         push @$cmd,  '-vnc', "unix:$socket,password";
3429     } else {
3430         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3431         push @$cmd, '-nographic';
3432     }
3433
3434     # time drift fix
3435     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3436     my $useLocaltime = $conf->{localtime};
3437
3438     if ($winversion >= 5) { # windows
3439         $useLocaltime = 1 if !defined($conf->{localtime});
3440
3441         # use time drift fix when acpi is enabled
3442         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3443             $tdf = 1 if !defined($conf->{tdf});
3444         }
3445     }
3446
3447     if ($winversion >= 6) {
3448         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3449         push @$cmd, '-no-hpet';
3450     }
3451
3452     push @$rtcFlags, 'driftfix=slew' if $tdf;
3453
3454     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3455         push @$rtcFlags, "base=$conf->{startdate}";
3456     } elsif ($useLocaltime) {
3457         push @$rtcFlags, 'base=localtime';
3458     }
3459
3460     if ($forcecpu) {
3461         push @$cmd, '-cpu', $forcecpu;
3462     } else {
3463         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3464     }
3465
3466     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3467
3468     push @$cmd, '-S' if $conf->{freeze};
3469
3470     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3471
3472     my $guest_agent = parse_guest_agent($conf);
3473
3474     if ($guest_agent->{enabled}) {
3475         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3476         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3477
3478         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3479             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3480             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3481             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3482         } elsif ($guest_agent->{type} eq 'isa') {
3483             push @$devices, '-device', "isa-serial,chardev=qga0";
3484         }
3485     }
3486
3487     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3488     if ($rng && $version_guard->(4, 1, 2)) {
3489         check_rng_source($rng->{source});
3490
3491         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3492         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3493         my $limiter_str = "";
3494         if ($max_bytes) {
3495             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3496         }
3497
3498         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3499         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3500         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3501     }
3502
3503     my $spice_port;
3504
3505     if ($qxlnum) {
3506         if ($qxlnum > 1) {
3507             if ($winversion){
3508                 for (my $i = 1; $i < $qxlnum; $i++){
3509                     push @$devices, '-device', print_vga_device(
3510                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3511                 }
3512             } else {
3513                 # assume other OS works like Linux
3514                 my ($ram, $vram) = ("134217728", "67108864");
3515                 if ($vga->{memory}) {
3516                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3517                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3518                 }
3519                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3520                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3521             }
3522         }
3523
3524         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3525
3526         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3527         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3528         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3529
3530         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3531         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3532         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3533
3534         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3535         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3536
3537         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3538         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3539         if ($spice_enhancement->{foldersharing}) {
3540             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3541             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3542         }
3543
3544         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3545         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3546             if $spice_enhancement->{videostreaming};
3547
3548         push @$devices, '-spice', "$spice_opts";
3549     }
3550
3551     # enable balloon by default, unless explicitly disabled
3552     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3553         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3554         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3555     }
3556
3557     if ($conf->{watchdog}) {
3558         my $wdopts = parse_watchdog($conf->{watchdog});
3559         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3560         my $watchdog = $wdopts->{model} || 'i6300esb';
3561         push @$devices, '-device', "$watchdog$pciaddr";
3562         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3563     }
3564
3565     my $vollist = [];
3566     my $scsicontroller = {};
3567     my $ahcicontroller = {};
3568     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3569
3570     # Add iscsi initiator name if available
3571     if (my $initiator = get_initiator_name()) {
3572         push @$devices, '-iscsi', "initiator-name=$initiator";
3573     }
3574
3575     PVE::QemuConfig->foreach_volume($conf, sub {
3576         my ($ds, $drive) = @_;
3577
3578         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3579             push @$vollist, $drive->{file};
3580         }
3581
3582         # ignore efidisk here, already added in bios/fw handling code above
3583         return if $drive->{interface} eq 'efidisk';
3584
3585         $use_virtio = 1 if $ds =~ m/^virtio/;
3586
3587         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3588
3589         if ($drive->{interface} eq 'virtio'){
3590            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3591         }
3592
3593         if ($drive->{interface} eq 'scsi') {
3594
3595             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3596
3597             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3598                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3599
3600             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3601             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3602
3603             my $iothread = '';
3604             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3605                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3606                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3607             } elsif ($drive->{iothread}) {
3608                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3609             }
3610
3611             my $queues = '';
3612             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3613                 $queues = ",num_queues=$drive->{queues}";
3614             }
3615
3616             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3617                 if !$scsicontroller->{$controller};
3618             $scsicontroller->{$controller}=1;
3619         }
3620
3621         if ($drive->{interface} eq 'sata') {
3622             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3623             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3624             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3625                 if !$ahcicontroller->{$controller};
3626             $ahcicontroller->{$controller}=1;
3627         }
3628
3629         my $pbs_conf = $pbs_backing->{$ds};
3630         my $pbs_name = undef;
3631         if ($pbs_conf) {
3632             $pbs_name = "drive-$ds-pbs";
3633             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3634         }
3635
3636         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3637         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3638
3639         push @$devices, '-drive',$drive_cmd;
3640         push @$devices, '-device', print_drivedevice_full(
3641             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3642     });
3643
3644     for (my $i = 0; $i < $MAX_NETS; $i++) {
3645         my $netname = "net$i";
3646
3647         next if !$conf->{$netname};
3648         my $d = parse_net($conf->{$netname});
3649         next if !$d;
3650
3651         $use_virtio = 1 if $d->{model} eq 'virtio';
3652
3653         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3654
3655         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3656         push @$devices, '-netdev', $netdevfull;
3657
3658         my $netdevicefull = print_netdevice_full(
3659             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3660
3661         push @$devices, '-device', $netdevicefull;
3662     }
3663
3664     if ($conf->{ivshmem}) {
3665         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3666
3667         my $bus;
3668         if ($q35) {
3669             $bus = print_pcie_addr("ivshmem");
3670         } else {
3671             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3672         }
3673
3674         my $ivshmem_name = $ivshmem->{name} // $vmid;
3675         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3676
3677         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3678         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3679             .",size=$ivshmem->{size}M";
3680     }
3681
3682     # pci.4 is nested in pci.1
3683     $bridges->{1} = 1 if $bridges->{4};
3684
3685     if (!$q35) {
3686         # add pci bridges
3687         if (min_version($machine_version, 2, 3)) {
3688            $bridges->{1} = 1;
3689            $bridges->{2} = 1;
3690         }
3691
3692         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3693
3694     }
3695
3696     for my $k (sort {$b cmp $a} keys %$bridges) {
3697         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3698
3699         my $k_name = $k;
3700         if ($k == 2 && $legacy_igd) {
3701             $k_name = "$k-igd";
3702         }
3703         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3704
3705         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3706         if ($q35) {
3707             # add after -readconfig pve-q35.cfg
3708             splice @$devices, 2, 0, '-device', $devstr;
3709         } else {
3710             unshift @$devices, '-device', $devstr if $k > 0;
3711         }
3712     }
3713
3714     if (!$kvm) {
3715         push @$machineFlags, 'accel=tcg';
3716     }
3717
3718     my $machine_type_min = $machine_type;
3719     if ($add_pve_version) {
3720         $machine_type_min =~ s/\+pve\d+$//;
3721         $machine_type_min .= "+pve$required_pve_version";
3722     }
3723     push @$machineFlags, "type=${machine_type_min}";
3724
3725     push @$cmd, @$devices;
3726     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3727     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3728     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3729
3730     if (my $vmstate = $conf->{vmstate}) {
3731         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3732         push @$vollist, $vmstate;
3733         push @$cmd, '-loadstate', $statepath;
3734         print "activating and using '$vmstate' as vmstate\n";
3735     }
3736
3737     # add custom args
3738     if ($conf->{args}) {
3739         my $aa = PVE::Tools::split_args($conf->{args});
3740         push @$cmd, @$aa;
3741     }
3742
3743     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3744 }
3745
3746 sub check_rng_source {
3747     my ($source) = @_;
3748
3749     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3750     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3751         if ! -e $source;
3752
3753     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3754     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3755         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3756         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3757         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3758             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3759             ." to the host.\n";
3760     }
3761 }
3762
3763 sub spice_port {
3764     my ($vmid) = @_;
3765
3766     my $res = mon_cmd($vmid, 'query-spice');
3767
3768     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3769 }
3770
3771 sub vm_devices_list {
3772     my ($vmid) = @_;
3773
3774     my $res = mon_cmd($vmid, 'query-pci');
3775     my $devices_to_check = [];
3776     my $devices = {};
3777     foreach my $pcibus (@$res) {
3778         push @$devices_to_check, @{$pcibus->{devices}},
3779     }
3780
3781     while (@$devices_to_check) {
3782         my $to_check = [];
3783         for my $d (@$devices_to_check) {
3784             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3785             next if !$d->{'pci_bridge'};
3786
3787             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3788             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3789         }
3790         $devices_to_check = $to_check;
3791     }
3792
3793     my $resblock = mon_cmd($vmid, 'query-block');
3794     foreach my $block (@$resblock) {
3795         if($block->{device} =~ m/^drive-(\S+)/){
3796                 $devices->{$1} = 1;
3797         }
3798     }
3799
3800     my $resmice = mon_cmd($vmid, 'query-mice');
3801     foreach my $mice (@$resmice) {
3802         if ($mice->{name} eq 'QEMU HID Tablet') {
3803             $devices->{tablet} = 1;
3804             last;
3805         }
3806     }
3807
3808     # for usb devices there is no query-usb
3809     # but we can iterate over the entries in
3810     # qom-list path=/machine/peripheral
3811     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3812     foreach my $per (@$resperipheral) {
3813         if ($per->{name} =~ m/^usb\d+$/) {
3814             $devices->{$per->{name}} = 1;
3815         }
3816     }
3817
3818     return $devices;
3819 }
3820
3821 sub vm_deviceplug {
3822     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3823
3824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3825
3826     my $devices_list = vm_devices_list($vmid);
3827     return 1 if defined($devices_list->{$deviceid});
3828
3829     # add PCI bridge if we need it for the device
3830     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3831
3832     if ($deviceid eq 'tablet') {
3833
3834         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3835
3836     } elsif ($deviceid eq 'keyboard') {
3837
3838         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3839
3840     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3841
3842         die "usb hotplug currently not reliable\n";
3843         # since we can't reliably hot unplug all added usb devices and usb
3844         # passthrough breaks live migration we disable usb hotplugging for now
3845         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3846
3847     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3848
3849         qemu_iothread_add($vmid, $deviceid, $device);
3850
3851         qemu_driveadd($storecfg, $vmid, $device);
3852         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3853
3854         qemu_deviceadd($vmid, $devicefull);
3855         eval { qemu_deviceaddverify($vmid, $deviceid); };
3856         if (my $err = $@) {
3857             eval { qemu_drivedel($vmid, $deviceid); };
3858             warn $@ if $@;
3859             die $err;
3860         }
3861
3862     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3863
3864
3865         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3866         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3867         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3868
3869         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3870
3871         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3872             qemu_iothread_add($vmid, $deviceid, $device);
3873             $devicefull .= ",iothread=iothread-$deviceid";
3874         }
3875
3876         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3877             $devicefull .= ",num_queues=$device->{queues}";
3878         }
3879
3880         qemu_deviceadd($vmid, $devicefull);
3881         qemu_deviceaddverify($vmid, $deviceid);
3882
3883     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3884
3885         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3886         qemu_driveadd($storecfg, $vmid, $device);
3887
3888         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3889         eval { qemu_deviceadd($vmid, $devicefull); };
3890         if (my $err = $@) {
3891             eval { qemu_drivedel($vmid, $deviceid); };
3892             warn $@ if $@;
3893             die $err;
3894         }
3895
3896     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3897
3898         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3899
3900         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3901         my $use_old_bios_files = undef;
3902         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3903
3904         my $netdevicefull = print_netdevice_full(
3905             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3906         qemu_deviceadd($vmid, $netdevicefull);
3907         eval {
3908             qemu_deviceaddverify($vmid, $deviceid);
3909             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3910         };
3911         if (my $err = $@) {
3912             eval { qemu_netdevdel($vmid, $deviceid); };
3913             warn $@ if $@;
3914             die $err;
3915         }
3916
3917     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3918
3919         my $bridgeid = $2;
3920         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3921         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3922
3923         qemu_deviceadd($vmid, $devicefull);
3924         qemu_deviceaddverify($vmid, $deviceid);
3925
3926     } else {
3927         die "can't hotplug device '$deviceid'\n";
3928     }
3929
3930     return 1;
3931 }
3932
3933 # fixme: this should raise exceptions on error!
3934 sub vm_deviceunplug {
3935     my ($vmid, $conf, $deviceid) = @_;
3936
3937     my $devices_list = vm_devices_list($vmid);
3938     return 1 if !defined($devices_list->{$deviceid});
3939
3940     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3941     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3942
3943     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3944
3945         qemu_devicedel($vmid, $deviceid);
3946
3947     } elsif ($deviceid =~ m/^usb\d+$/) {
3948
3949         die "usb hotplug currently not reliable\n";
3950         # when unplugging usb devices this way, there may be remaining usb
3951         # controllers/hubs so we disable it for now
3952         #qemu_devicedel($vmid, $deviceid);
3953         #qemu_devicedelverify($vmid, $deviceid);
3954
3955     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3956
3957         qemu_devicedel($vmid, $deviceid);
3958         qemu_devicedelverify($vmid, $deviceid);
3959         qemu_drivedel($vmid, $deviceid);
3960         qemu_iothread_del($conf, $vmid, $deviceid);
3961
3962     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3963
3964         qemu_devicedel($vmid, $deviceid);
3965         qemu_devicedelverify($vmid, $deviceid);
3966         qemu_iothread_del($conf, $vmid, $deviceid);
3967
3968     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3969
3970         qemu_devicedel($vmid, $deviceid);
3971         qemu_drivedel($vmid, $deviceid);
3972         qemu_deletescsihw($conf, $vmid, $deviceid);
3973
3974     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3975
3976         qemu_devicedel($vmid, $deviceid);
3977         qemu_devicedelverify($vmid, $deviceid);
3978         qemu_netdevdel($vmid, $deviceid);
3979
3980     } else {
3981         die "can't unplug device '$deviceid'\n";
3982     }
3983
3984     return 1;
3985 }
3986
3987 sub qemu_deviceadd {
3988     my ($vmid, $devicefull) = @_;
3989
3990     $devicefull = "driver=".$devicefull;
3991     my %options =  split(/[=,]/, $devicefull);
3992
3993     mon_cmd($vmid, "device_add" , %options);
3994 }
3995
3996 sub qemu_devicedel {
3997     my ($vmid, $deviceid) = @_;
3998
3999     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4000 }
4001
4002 sub qemu_iothread_add {
4003     my($vmid, $deviceid, $device) = @_;
4004
4005     if ($device->{iothread}) {
4006         my $iothreads = vm_iothreads_list($vmid);
4007         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4008     }
4009 }
4010
4011 sub qemu_iothread_del {
4012     my($conf, $vmid, $deviceid) = @_;
4013
4014     my $confid = $deviceid;
4015     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4016         $confid = 'scsi' . $1;
4017     }
4018     my $device = parse_drive($confid, $conf->{$confid});
4019     if ($device->{iothread}) {
4020         my $iothreads = vm_iothreads_list($vmid);
4021         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4022     }
4023 }
4024
4025 sub qemu_objectadd {
4026     my($vmid, $objectid, $qomtype) = @_;
4027
4028     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4029
4030     return 1;
4031 }
4032
4033 sub qemu_objectdel {
4034     my($vmid, $objectid) = @_;
4035
4036     mon_cmd($vmid, "object-del", id => $objectid);
4037
4038     return 1;
4039 }
4040
4041 sub qemu_driveadd {
4042     my ($storecfg, $vmid, $device) = @_;
4043
4044     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4045     $drive =~ s/\\/\\\\/g;
4046     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4047
4048     # If the command succeeds qemu prints: "OK"
4049     return 1 if $ret =~ m/OK/s;
4050
4051     die "adding drive failed: $ret\n";
4052 }
4053
4054 sub qemu_drivedel {
4055     my($vmid, $deviceid) = @_;
4056
4057     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4058     $ret =~ s/^\s+//;
4059
4060     return 1 if $ret eq "";
4061
4062     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4063     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4064
4065     die "deleting drive $deviceid failed : $ret\n";
4066 }
4067
4068 sub qemu_deviceaddverify {
4069     my ($vmid, $deviceid) = @_;
4070
4071     for (my $i = 0; $i <= 5; $i++) {
4072          my $devices_list = vm_devices_list($vmid);
4073          return 1 if defined($devices_list->{$deviceid});
4074          sleep 1;
4075     }
4076
4077     die "error on hotplug device '$deviceid'\n";
4078 }
4079
4080
4081 sub qemu_devicedelverify {
4082     my ($vmid, $deviceid) = @_;
4083
4084     # need to verify that the device is correctly removed as device_del
4085     # is async and empty return is not reliable
4086
4087     for (my $i = 0; $i <= 5; $i++) {
4088          my $devices_list = vm_devices_list($vmid);
4089          return 1 if !defined($devices_list->{$deviceid});
4090          sleep 1;
4091     }
4092
4093     die "error on hot-unplugging device '$deviceid'\n";
4094 }
4095
4096 sub qemu_findorcreatescsihw {
4097     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4098
4099     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4100
4101     my $scsihwid="$controller_prefix$controller";
4102     my $devices_list = vm_devices_list($vmid);
4103
4104     if(!defined($devices_list->{$scsihwid})) {
4105         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4106     }
4107
4108     return 1;
4109 }
4110
4111 sub qemu_deletescsihw {
4112     my ($conf, $vmid, $opt) = @_;
4113
4114     my $device = parse_drive($opt, $conf->{$opt});
4115
4116     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4117         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4118         return 1;
4119     }
4120
4121     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4122
4123     my $devices_list = vm_devices_list($vmid);
4124     foreach my $opt (keys %{$devices_list}) {
4125         if (is_valid_drivename($opt)) {
4126             my $drive = parse_drive($opt, $conf->{$opt});
4127             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4128                 return 1;
4129             }
4130         }
4131     }
4132
4133     my $scsihwid="scsihw$controller";
4134
4135     vm_deviceunplug($vmid, $conf, $scsihwid);
4136
4137     return 1;
4138 }
4139
4140 sub qemu_add_pci_bridge {
4141     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4142
4143     my $bridges = {};
4144
4145     my $bridgeid;
4146
4147     print_pci_addr($device, $bridges, $arch, $machine_type);
4148
4149     while (my ($k, $v) = each %$bridges) {
4150         $bridgeid = $k;
4151     }
4152     return 1 if !defined($bridgeid) || $bridgeid < 1;
4153
4154     my $bridge = "pci.$bridgeid";
4155     my $devices_list = vm_devices_list($vmid);
4156
4157     if (!defined($devices_list->{$bridge})) {
4158         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4159     }
4160
4161     return 1;
4162 }
4163
4164 sub qemu_set_link_status {
4165     my ($vmid, $device, $up) = @_;
4166
4167     mon_cmd($vmid, "set_link", name => $device,
4168                up => $up ? JSON::true : JSON::false);
4169 }
4170
4171 sub qemu_netdevadd {
4172     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4173
4174     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4175     my %options =  split(/[=,]/, $netdev);
4176
4177     if (defined(my $vhost = $options{vhost})) {
4178         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4179     }
4180
4181     if (defined(my $queues = $options{queues})) {
4182         $options{queues} = $queues + 0;
4183     }
4184
4185     mon_cmd($vmid, "netdev_add",  %options);
4186     return 1;
4187 }
4188
4189 sub qemu_netdevdel {
4190     my ($vmid, $deviceid) = @_;
4191
4192     mon_cmd($vmid, "netdev_del", id => $deviceid);
4193 }
4194
4195 sub qemu_usb_hotplug {
4196     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4197
4198     return if !$device;
4199
4200     # remove the old one first
4201     vm_deviceunplug($vmid, $conf, $deviceid);
4202
4203     # check if xhci controller is necessary and available
4204     if ($device->{usb3}) {
4205
4206         my $devicelist = vm_devices_list($vmid);
4207
4208         if (!$devicelist->{xhci}) {
4209             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4210             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4211         }
4212     }
4213     my $d = parse_usb_device($device->{host});
4214     $d->{usb3} = $device->{usb3};
4215
4216     # add the new one
4217     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4218 }
4219
4220 sub qemu_cpu_hotplug {
4221     my ($vmid, $conf, $vcpus) = @_;
4222
4223     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4224
4225     my $sockets = 1;
4226     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4227     $sockets = $conf->{sockets} if  $conf->{sockets};
4228     my $cores = $conf->{cores} || 1;
4229     my $maxcpus = $sockets * $cores;
4230
4231     $vcpus = $maxcpus if !$vcpus;
4232
4233     die "you can't add more vcpus than maxcpus\n"
4234         if $vcpus > $maxcpus;
4235
4236     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4237
4238     if ($vcpus < $currentvcpus) {
4239
4240         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4241
4242             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4243                 qemu_devicedel($vmid, "cpu$i");
4244                 my $retry = 0;
4245                 my $currentrunningvcpus = undef;
4246                 while (1) {
4247                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4248                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4249                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4250                     $retry++;
4251                     sleep 1;
4252                 }
4253                 #update conf after each succesfull cpu unplug
4254                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4255                 PVE::QemuConfig->write_config($vmid, $conf);
4256             }
4257         } else {
4258             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4259         }
4260
4261         return;
4262     }
4263
4264     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4265     die "vcpus in running vm does not match its configuration\n"
4266         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4267
4268     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4269
4270         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4271             my $cpustr = print_cpu_device($conf, $i);
4272             qemu_deviceadd($vmid, $cpustr);
4273
4274             my $retry = 0;
4275             my $currentrunningvcpus = undef;
4276             while (1) {
4277                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4278                 last if scalar(@{$currentrunningvcpus}) == $i;
4279                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4280                 sleep 1;
4281                 $retry++;
4282             }
4283             #update conf after each succesfull cpu hotplug
4284             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4285             PVE::QemuConfig->write_config($vmid, $conf);
4286         }
4287     } else {
4288
4289         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4290             mon_cmd($vmid, "cpu-add", id => int($i));
4291         }
4292     }
4293 }
4294
4295 sub qemu_block_set_io_throttle {
4296     my ($vmid, $deviceid,
4297         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4298         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4299         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4300         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4301
4302     return if !check_running($vmid) ;
4303
4304     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4305         bps => int($bps),
4306         bps_rd => int($bps_rd),
4307         bps_wr => int($bps_wr),
4308         iops => int($iops),
4309         iops_rd => int($iops_rd),
4310         iops_wr => int($iops_wr),
4311         bps_max => int($bps_max),
4312         bps_rd_max => int($bps_rd_max),
4313         bps_wr_max => int($bps_wr_max),
4314         iops_max => int($iops_max),
4315         iops_rd_max => int($iops_rd_max),
4316         iops_wr_max => int($iops_wr_max),
4317         bps_max_length => int($bps_max_length),
4318         bps_rd_max_length => int($bps_rd_max_length),
4319         bps_wr_max_length => int($bps_wr_max_length),
4320         iops_max_length => int($iops_max_length),
4321         iops_rd_max_length => int($iops_rd_max_length),
4322         iops_wr_max_length => int($iops_wr_max_length),
4323     );
4324
4325 }
4326
4327 sub qemu_block_resize {
4328     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4329
4330     my $running = check_running($vmid);
4331
4332     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4333
4334     return if !$running;
4335
4336     my $padding = (1024 - $size % 1024) % 1024;
4337     $size = $size + $padding;
4338
4339     mon_cmd(
4340         $vmid,
4341         "block_resize",
4342         device => $deviceid,
4343         size => int($size),
4344         timeout => 60,
4345     );
4346 }
4347
4348 sub qemu_volume_snapshot {
4349     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4350
4351     my $running = check_running($vmid);
4352
4353     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4354         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4355     } else {
4356         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4357     }
4358 }
4359
4360 sub qemu_volume_snapshot_delete {
4361     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4362
4363     my $running = check_running($vmid);
4364
4365     if($running) {
4366
4367         $running = undef;
4368         my $conf = PVE::QemuConfig->load_config($vmid);
4369         PVE::QemuConfig->foreach_volume($conf, sub {
4370             my ($ds, $drive) = @_;
4371             $running = 1 if $drive->{file} eq $volid;
4372         });
4373     }
4374
4375     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4376         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4377     } else {
4378         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4379     }
4380 }
4381
4382 sub set_migration_caps {
4383     my ($vmid, $savevm) = @_;
4384
4385     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4386
4387     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4388     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4389
4390     my $cap_ref = [];
4391
4392     my $enabled_cap = {
4393         "auto-converge" => 1,
4394         "xbzrle" => 1,
4395         "x-rdma-pin-all" => 0,
4396         "zero-blocks" => 0,
4397         "compress" => 0,
4398         "dirty-bitmaps" => $dirty_bitmaps,
4399     };
4400
4401     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4402
4403     for my $supported_capability (@$supported_capabilities) {
4404         push @$cap_ref, {
4405             capability => $supported_capability->{capability},
4406             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4407         };
4408     }
4409
4410     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4411 }
4412
4413 sub foreach_volid {
4414     my ($conf, $func, @param) = @_;
4415
4416     my $volhash = {};
4417
4418     my $test_volid = sub {
4419         my ($key, $drive, $snapname) = @_;
4420
4421         my $volid = $drive->{file};
4422         return if !$volid;
4423
4424         $volhash->{$volid}->{cdrom} //= 1;
4425         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4426
4427         my $replicate = $drive->{replicate} // 1;
4428         $volhash->{$volid}->{replicate} //= 0;
4429         $volhash->{$volid}->{replicate} = 1 if $replicate;
4430
4431         $volhash->{$volid}->{shared} //= 0;
4432         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4433
4434         $volhash->{$volid}->{referenced_in_config} //= 0;
4435         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4436
4437         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4438             if defined($snapname);
4439
4440         my $size = $drive->{size};
4441         $volhash->{$volid}->{size} //= $size if $size;
4442
4443         $volhash->{$volid}->{is_vmstate} //= 0;
4444         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4445
4446         $volhash->{$volid}->{is_unused} //= 0;
4447         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4448     };
4449
4450     my $include_opts = {
4451         extra_keys => ['vmstate'],
4452         include_unused => 1,
4453     };
4454
4455     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4456     foreach my $snapname (keys %{$conf->{snapshots}}) {
4457         my $snap = $conf->{snapshots}->{$snapname};
4458         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4459     }
4460
4461     foreach my $volid (keys %$volhash) {
4462         &$func($volid, $volhash->{$volid}, @param);
4463     }
4464 }
4465
4466 my $fast_plug_option = {
4467     'lock' => 1,
4468     'name' => 1,
4469     'onboot' => 1,
4470     'shares' => 1,
4471     'startup' => 1,
4472     'description' => 1,
4473     'protection' => 1,
4474     'vmstatestorage' => 1,
4475     'hookscript' => 1,
4476     'tags' => 1,
4477 };
4478
4479 # hotplug changes in [PENDING]
4480 # $selection hash can be used to only apply specified options, for
4481 # example: { cores => 1 } (only apply changed 'cores')
4482 # $errors ref is used to return error messages
4483 sub vmconfig_hotplug_pending {
4484     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4485
4486     my $defaults = load_defaults();
4487     my $arch = get_vm_arch($conf);
4488     my $machine_type = get_vm_machine($conf, undef, $arch);
4489
4490     # commit values which do not have any impact on running VM first
4491     # Note: those option cannot raise errors, we we do not care about
4492     # $selection and always apply them.
4493
4494     my $add_error = sub {
4495         my ($opt, $msg) = @_;
4496         $errors->{$opt} = "hotplug problem - $msg";
4497     };
4498
4499     my $changes = 0;
4500     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4501         if ($fast_plug_option->{$opt}) {
4502             $conf->{$opt} = $conf->{pending}->{$opt};
4503             delete $conf->{pending}->{$opt};
4504             $changes = 1;
4505         }
4506     }
4507
4508     if ($changes) {
4509         PVE::QemuConfig->write_config($vmid, $conf);
4510     }
4511
4512     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4513
4514     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4515     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4516     foreach my $opt (sort keys %$pending_delete_hash) {
4517         next if $selection && !$selection->{$opt};
4518         my $force = $pending_delete_hash->{$opt}->{force};
4519         eval {
4520             if ($opt eq 'hotplug') {
4521                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4522             } elsif ($opt eq 'tablet') {
4523                 die "skip\n" if !$hotplug_features->{usb};
4524                 if ($defaults->{tablet}) {
4525                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4526                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4527                         if $arch eq 'aarch64';
4528                 } else {
4529                     vm_deviceunplug($vmid, $conf, 'tablet');
4530                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4531                 }
4532             } elsif ($opt =~ m/^usb\d+/) {
4533                 die "skip\n";
4534                 # since we cannot reliably hot unplug usb devices we are disabling it
4535                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4536                 #vm_deviceunplug($vmid, $conf, $opt);
4537             } elsif ($opt eq 'vcpus') {
4538                 die "skip\n" if !$hotplug_features->{cpu};
4539                 qemu_cpu_hotplug($vmid, $conf, undef);
4540             } elsif ($opt eq 'balloon') {
4541                 # enable balloon device is not hotpluggable
4542                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4543                 # here we reset the ballooning value to memory
4544                 my $balloon = $conf->{memory} || $defaults->{memory};
4545                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4546             } elsif ($fast_plug_option->{$opt}) {
4547                 # do nothing
4548             } elsif ($opt =~ m/^net(\d+)$/) {
4549                 die "skip\n" if !$hotplug_features->{network};
4550                 vm_deviceunplug($vmid, $conf, $opt);
4551             } elsif (is_valid_drivename($opt)) {
4552                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4553                 vm_deviceunplug($vmid, $conf, $opt);
4554                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4555             } elsif ($opt =~ m/^memory$/) {
4556                 die "skip\n" if !$hotplug_features->{memory};
4557                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4558             } elsif ($opt eq 'cpuunits') {
4559                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4560             } elsif ($opt eq 'cpulimit') {
4561                 $cgroup->change_cpu_quota(-1, 100000);
4562             } else {
4563                 die "skip\n";
4564             }
4565         };
4566         if (my $err = $@) {
4567             &$add_error($opt, $err) if $err ne "skip\n";
4568         } else {
4569             delete $conf->{$opt};
4570             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4571         }
4572     }
4573
4574     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4575     $apply_pending_cloudinit = sub {
4576         return if $apply_pending_cloudinit_done; # once is enough
4577         $apply_pending_cloudinit_done = 1; # once is enough
4578
4579         my ($key, $value) = @_;
4580
4581         my @cloudinit_opts = keys %$confdesc_cloudinit;
4582         foreach my $opt (keys %{$conf->{pending}}) {
4583             next if !grep { $_ eq $opt } @cloudinit_opts;
4584             $conf->{$opt} = delete $conf->{pending}->{$opt};
4585         }
4586
4587         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4588         foreach my $opt (sort keys %$pending_delete_hash) {
4589             next if !grep { $_ eq $opt } @cloudinit_opts;
4590             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4591             delete $conf->{$opt};
4592         }
4593
4594         my $new_conf = { %$conf };
4595         $new_conf->{$key} = $value;
4596         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4597     };
4598
4599     foreach my $opt (keys %{$conf->{pending}}) {
4600         next if $selection && !$selection->{$opt};
4601         my $value = $conf->{pending}->{$opt};
4602         eval {
4603             if ($opt eq 'hotplug') {
4604                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4605             } elsif ($opt eq 'tablet') {
4606                 die "skip\n" if !$hotplug_features->{usb};
4607                 if ($value == 1) {
4608                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4609                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4610                         if $arch eq 'aarch64';
4611                 } elsif ($value == 0) {
4612                     vm_deviceunplug($vmid, $conf, 'tablet');
4613                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4614                 }
4615             } elsif ($opt =~ m/^usb\d+$/) {
4616                 die "skip\n";
4617                 # since we cannot reliably hot unplug usb devices we disable it for now
4618                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4619                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4620                 #die "skip\n" if !$d;
4621                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4622             } elsif ($opt eq 'vcpus') {
4623                 die "skip\n" if !$hotplug_features->{cpu};
4624                 qemu_cpu_hotplug($vmid, $conf, $value);
4625             } elsif ($opt eq 'balloon') {
4626                 # enable/disable balloning device is not hotpluggable
4627                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4628                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4629                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4630
4631                 # allow manual ballooning if shares is set to zero
4632                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4633                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4634                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4635                 }
4636             } elsif ($opt =~ m/^net(\d+)$/) {
4637                 # some changes can be done without hotplug
4638                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4639                                     $vmid, $opt, $value, $arch, $machine_type);
4640             } elsif (is_valid_drivename($opt)) {
4641                 die "skip\n" if $opt eq 'efidisk0';
4642                 # some changes can be done without hotplug
4643                 my $drive = parse_drive($opt, $value);
4644                 if (drive_is_cloudinit($drive)) {
4645                     &$apply_pending_cloudinit($opt, $value);
4646                 }
4647                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4648                                      $vmid, $opt, $value, $arch, $machine_type);
4649             } elsif ($opt =~ m/^memory$/) { #dimms
4650                 die "skip\n" if !$hotplug_features->{memory};
4651                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4652             } elsif ($opt eq 'cpuunits') {
4653                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4654             } elsif ($opt eq 'cpulimit') {
4655                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4656                 $cgroup->change_cpu_quota($cpulimit, 100000);
4657             } else {
4658                 die "skip\n";  # skip non-hot-pluggable options
4659             }
4660         };
4661         if (my $err = $@) {
4662             &$add_error($opt, $err) if $err ne "skip\n";
4663         } else {
4664             $conf->{$opt} = $value;
4665             delete $conf->{pending}->{$opt};
4666         }
4667     }
4668
4669     PVE::QemuConfig->write_config($vmid, $conf);
4670 }
4671
4672 sub try_deallocate_drive {
4673     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4674
4675     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4676         my $volid = $drive->{file};
4677         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4678             my $sid = PVE::Storage::parse_volume_id($volid);
4679             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4680
4681             # check if the disk is really unused
4682             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4683                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4684             PVE::Storage::vdisk_free($storecfg, $volid);
4685             return 1;
4686         } else {
4687             # If vm is not owner of this disk remove from config
4688             return 1;
4689         }
4690     }
4691
4692     return;
4693 }
4694
4695 sub vmconfig_delete_or_detach_drive {
4696     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4697
4698     my $drive = parse_drive($opt, $conf->{$opt});
4699
4700     my $rpcenv = PVE::RPCEnvironment::get();
4701     my $authuser = $rpcenv->get_user();
4702
4703     if ($force) {
4704         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4705         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4706     } else {
4707         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4708     }
4709 }
4710
4711
4712
4713 sub vmconfig_apply_pending {
4714     my ($vmid, $conf, $storecfg, $errors) = @_;
4715
4716     my $add_apply_error = sub {
4717         my ($opt, $msg) = @_;
4718         my $err_msg = "unable to apply pending change $opt : $msg";
4719         $errors->{$opt} = $err_msg;
4720         warn $err_msg;
4721     };
4722
4723     # cold plug
4724
4725     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4726     foreach my $opt (sort keys %$pending_delete_hash) {
4727         my $force = $pending_delete_hash->{$opt}->{force};
4728         eval {
4729             if ($opt =~ m/^unused/) {
4730                 die "internal error";
4731             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4732                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4733             }
4734         };
4735         if (my $err = $@) {
4736             $add_apply_error->($opt, $err);
4737         } else {
4738             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4739             delete $conf->{$opt};
4740         }
4741     }
4742
4743     PVE::QemuConfig->cleanup_pending($conf);
4744
4745     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4746         next if $opt eq 'delete'; # just to be sure
4747         eval {
4748             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4749                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4750             }
4751         };
4752         if (my $err = $@) {
4753             $add_apply_error->($opt, $err);
4754         } else {
4755             $conf->{$opt} = delete $conf->{pending}->{$opt};
4756         }
4757     }
4758
4759     # write all changes at once to avoid unnecessary i/o
4760     PVE::QemuConfig->write_config($vmid, $conf);
4761 }
4762
4763 sub vmconfig_update_net {
4764     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4765
4766     my $newnet = parse_net($value);
4767
4768     if ($conf->{$opt}) {
4769         my $oldnet = parse_net($conf->{$opt});
4770
4771         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4772             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4773             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4774             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4775
4776             # for non online change, we try to hot-unplug
4777             die "skip\n" if !$hotplug;
4778             vm_deviceunplug($vmid, $conf, $opt);
4779         } else {
4780
4781             die "internal error" if $opt !~ m/net(\d+)/;
4782             my $iface = "tap${vmid}i$1";
4783
4784             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4785                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4786                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4787                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4788                 PVE::Network::tap_unplug($iface);
4789
4790                 if ($have_sdn) {
4791                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4792                 } else {
4793                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4794                 }
4795             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4796                 # Rate can be applied on its own but any change above needs to
4797                 # include the rate in tap_plug since OVS resets everything.
4798                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4799             }
4800
4801             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4802                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4803             }
4804
4805             return 1;
4806         }
4807     }
4808
4809     if ($hotplug) {
4810         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4811     } else {
4812         die "skip\n";
4813     }
4814 }
4815
4816 sub vmconfig_update_disk {
4817     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4818
4819     my $drive = parse_drive($opt, $value);
4820
4821     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4822         my $media = $drive->{media} || 'disk';
4823         my $oldmedia = $old_drive->{media} || 'disk';
4824         die "unable to change media type\n" if $media ne $oldmedia;
4825
4826         if (!drive_is_cdrom($old_drive)) {
4827
4828             if ($drive->{file} ne $old_drive->{file}) {
4829
4830                 die "skip\n" if !$hotplug;
4831
4832                 # unplug and register as unused
4833                 vm_deviceunplug($vmid, $conf, $opt);
4834                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4835
4836             } else {
4837                 # update existing disk
4838
4839                 # skip non hotpluggable value
4840                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4841                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4842                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4843                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4844                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4845                     die "skip\n";
4846                 }
4847
4848                 # apply throttle
4849                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4850                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4851                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4852                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4853                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4854                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4855                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4856                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4857                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4858                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4859                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4860                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4861                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4862                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4863                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4864                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4865                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4866                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4867
4868                     qemu_block_set_io_throttle(
4869                         $vmid,"drive-$opt",
4870                         ($drive->{mbps} || 0)*1024*1024,
4871                         ($drive->{mbps_rd} || 0)*1024*1024,
4872                         ($drive->{mbps_wr} || 0)*1024*1024,
4873                          $drive->{iops} || 0,
4874                          $drive->{iops_rd} || 0,
4875                          $drive->{iops_wr} || 0,
4876                         ($drive->{mbps_max} || 0)*1024*1024,
4877                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4878                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4879                          $drive->{iops_max} || 0,
4880                          $drive->{iops_rd_max} || 0,
4881                          $drive->{iops_wr_max} || 0,
4882                          $drive->{bps_max_length} || 1,
4883                          $drive->{bps_rd_max_length} || 1,
4884                          $drive->{bps_wr_max_length} || 1,
4885                          $drive->{iops_max_length} || 1,
4886                          $drive->{iops_rd_max_length} || 1,
4887                          $drive->{iops_wr_max_length} || 1,
4888                     );
4889
4890                 }
4891
4892                 return 1;
4893             }
4894
4895         } else { # cdrom
4896
4897             if ($drive->{file} eq 'none') {
4898                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4899                 if (drive_is_cloudinit($old_drive)) {
4900                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4901                 }
4902             } else {
4903                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4904
4905                 # force eject if locked
4906                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4907
4908                 if ($path) {
4909                     mon_cmd($vmid, "blockdev-change-medium",
4910                         id => "$opt", filename => "$path");
4911                 }
4912             }
4913
4914             return 1;
4915         }
4916     }
4917
4918     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4919     # hotplug new disks
4920     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4921     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4922 }
4923
4924 # called in locked context by incoming migration
4925 sub vm_migrate_get_nbd_disks {
4926     my ($storecfg, $conf, $replicated_volumes) = @_;
4927
4928     my $local_volumes = {};
4929     PVE::QemuConfig->foreach_volume($conf, sub {
4930         my ($ds, $drive) = @_;
4931
4932         return if drive_is_cdrom($drive);
4933
4934         my $volid = $drive->{file};
4935
4936         return if !$volid;
4937
4938         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4939
4940         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4941         return if $scfg->{shared};
4942
4943         # replicated disks re-use existing state via bitmap
4944         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4945         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4946     });
4947     return $local_volumes;
4948 }
4949
4950 # called in locked context by incoming migration
4951 sub vm_migrate_alloc_nbd_disks {
4952     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4953
4954     my $format = undef;
4955
4956     my $nbd = {};
4957     foreach my $opt (sort keys %$source_volumes) {
4958         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4959
4960         if ($use_existing) {
4961             $nbd->{$opt}->{drivestr} = print_drive($drive);
4962             $nbd->{$opt}->{volid} = $volid;
4963             $nbd->{$opt}->{replicated} = 1;
4964             next;
4965         }
4966
4967         # If a remote storage is specified and the format of the original
4968         # volume is not available there, fall back to the default format.
4969         # Otherwise use the same format as the original.
4970         if (!$storagemap->{identity}) {
4971             $storeid = map_storage($storagemap, $storeid);
4972             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4973             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4974             my $fileFormat = qemu_img_format($scfg, $volname);
4975             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4976         } else {
4977             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4978             $format = qemu_img_format($scfg, $volname);
4979         }
4980
4981         my $size = $drive->{size} / 1024;
4982         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4983         my $newdrive = $drive;
4984         $newdrive->{format} = $format;
4985         $newdrive->{file} = $newvolid;
4986         my $drivestr = print_drive($newdrive);
4987         $nbd->{$opt}->{drivestr} = $drivestr;
4988         $nbd->{$opt}->{volid} = $newvolid;
4989     }
4990
4991     return $nbd;
4992 }
4993
4994 # see vm_start_nolock for parameters, additionally:
4995 # migrate_opts:
4996 #   storagemap = parsed storage map for allocating NBD disks
4997 sub vm_start {
4998     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4999
5000     return PVE::QemuConfig->lock_config($vmid, sub {
5001         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5002
5003         die "you can't start a vm if it's a template\n"
5004             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5005
5006         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5007         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5008
5009         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5010
5011         if ($has_backup_lock && $running) {
5012             # a backup is currently running, attempt to start the guest in the
5013             # existing QEMU instance
5014             return vm_resume($vmid);
5015         }
5016
5017         PVE::QemuConfig->check_lock($conf)
5018             if !($params->{skiplock} || $has_suspended_lock);
5019
5020         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5021
5022         die "VM $vmid already running\n" if $running;
5023
5024         if (my $storagemap = $migrate_opts->{storagemap}) {
5025             my $replicated = $migrate_opts->{replicated_volumes};
5026             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5027             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5028
5029             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5030                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5031             }
5032         }
5033
5034         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5035     });
5036 }
5037
5038
5039 # params:
5040 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5041 #   skiplock => 0/1, skip checking for config lock
5042 #   skiptemplate => 0/1, skip checking whether VM is template
5043 #   forcemachine => to force Qemu machine (rollback/migration)
5044 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5045 #   timeout => in seconds
5046 #   paused => start VM in paused state (backup)
5047 #   resume => resume from hibernation
5048 #   pbs-backing => {
5049 #      sata0 => {
5050 #         repository
5051 #         snapshot
5052 #         keyfile
5053 #         archive
5054 #      },
5055 #      virtio2 => ...
5056 #   }
5057 # migrate_opts:
5058 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5059 #   migratedfrom => source node
5060 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5061 #   network => CIDR of migration network
5062 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5063 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5064 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5065 sub vm_start_nolock {
5066     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5067
5068     my $statefile = $params->{statefile};
5069     my $resume = $params->{resume};
5070
5071     my $migratedfrom = $migrate_opts->{migratedfrom};
5072     my $migration_type = $migrate_opts->{type};
5073
5074     my $res = {};
5075
5076     # clean up leftover reboot request files
5077     eval { clear_reboot_request($vmid); };
5078     warn $@ if $@;
5079
5080     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5081         vmconfig_apply_pending($vmid, $conf, $storecfg);
5082         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5083     }
5084
5085     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5086
5087     my $defaults = load_defaults();
5088
5089     # set environment variable useful inside network script
5090     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5091
5092     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5093
5094     my $forcemachine = $params->{forcemachine};
5095     my $forcecpu = $params->{forcecpu};
5096     if ($resume) {
5097         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5098         $forcemachine = $conf->{runningmachine};
5099         $forcecpu = $conf->{runningcpu};
5100         print "Resuming suspended VM\n";
5101     }
5102
5103     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5104         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5105
5106     my $migration_ip;
5107     my $get_migration_ip = sub {
5108         my ($nodename) = @_;
5109
5110         return $migration_ip if defined($migration_ip);
5111
5112         my $cidr = $migrate_opts->{network};
5113
5114         if (!defined($cidr)) {
5115             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5116             $cidr = $dc_conf->{migration}->{network};
5117         }
5118
5119         if (defined($cidr)) {
5120             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5121
5122             die "could not get IP: no address configured on local " .
5123                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5124
5125             die "could not get IP: multiple addresses configured on local " .
5126                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5127
5128             $migration_ip = @$ips[0];
5129         }
5130
5131         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5132             if !defined($migration_ip);
5133
5134         return $migration_ip;
5135     };
5136
5137     my $migrate_uri;
5138     if ($statefile) {
5139         if ($statefile eq 'tcp') {
5140             my $localip = "localhost";
5141             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5142             my $nodename = nodename();
5143
5144             if (!defined($migration_type)) {
5145                 if (defined($datacenterconf->{migration}->{type})) {
5146                     $migration_type = $datacenterconf->{migration}->{type};
5147                 } else {
5148                     $migration_type = 'secure';
5149                 }
5150             }
5151
5152             if ($migration_type eq 'insecure') {
5153                 $localip = $get_migration_ip->($nodename);
5154                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5155             }
5156
5157             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5158             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5159             $migrate_uri = "tcp:${localip}:${migrate_port}";
5160             push @$cmd, '-incoming', $migrate_uri;
5161             push @$cmd, '-S';
5162
5163         } elsif ($statefile eq 'unix') {
5164             # should be default for secure migrations as a ssh TCP forward
5165             # tunnel is not deterministic reliable ready and fails regurarly
5166             # to set up in time, so use UNIX socket forwards
5167             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5168             unlink $socket_addr;
5169
5170             $migrate_uri = "unix:$socket_addr";
5171
5172             push @$cmd, '-incoming', $migrate_uri;
5173             push @$cmd, '-S';
5174
5175         } elsif (-e $statefile) {
5176             push @$cmd, '-loadstate', $statefile;
5177         } else {
5178             my $statepath = PVE::Storage::path($storecfg, $statefile);
5179             push @$vollist, $statefile;
5180             push @$cmd, '-loadstate', $statepath;
5181         }
5182     } elsif ($params->{paused}) {
5183         push @$cmd, '-S';
5184     }
5185
5186     # host pci devices
5187     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5188       my $d = parse_hostpci($conf->{"hostpci$i"});
5189       next if !$d;
5190       my $pcidevices = $d->{pciid};
5191       foreach my $pcidevice (@$pcidevices) {
5192             my $pciid = $pcidevice->{id};
5193
5194             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5195             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5196             die "no pci device info for device '$pciid'\n" if !$info;
5197
5198             if ($d->{mdev}) {
5199                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5200                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5201             } else {
5202                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5203                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5204                 die "can't reset PCI device '$pciid'\n"
5205                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5206             }
5207       }
5208     }
5209
5210     PVE::Storage::activate_volumes($storecfg, $vollist);
5211
5212     eval {
5213         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5214             outfunc => sub {}, errfunc => sub {});
5215     };
5216     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5217     # timeout should be more than enough here...
5218     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5219
5220     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5221                                               : $defaults->{cpuunits};
5222
5223     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5224     my %run_params = (
5225         timeout => $statefile ? undef : $start_timeout,
5226         umask => 0077,
5227         noerr => 1,
5228     );
5229
5230     # when migrating, prefix QEMU output so other side can pick up any
5231     # errors that might occur and show the user
5232     if ($migratedfrom) {
5233         $run_params{quiet} = 1;
5234         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5235     }
5236
5237     my %properties = (
5238         Slice => 'qemu.slice',
5239         KillMode => 'none'
5240     );
5241
5242     if (PVE::CGroup::cgroup_mode() == 2) {
5243         $properties{CPUWeight} = $cpuunits;
5244     } else {
5245         $properties{CPUShares} = $cpuunits;
5246     }
5247
5248     if (my $cpulimit = $conf->{cpulimit}) {
5249         $properties{CPUQuota} = int($cpulimit * 100);
5250     }
5251     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5252
5253     my $run_qemu = sub {
5254         PVE::Tools::run_fork sub {
5255             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5256
5257             my $exitcode = run_command($cmd, %run_params);
5258             die "QEMU exited with code $exitcode\n" if $exitcode;
5259         };
5260     };
5261
5262     if ($conf->{hugepages}) {
5263
5264         my $code = sub {
5265             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5266             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5267
5268             PVE::QemuServer::Memory::hugepages_mount();
5269             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5270
5271             eval { $run_qemu->() };
5272             if (my $err = $@) {
5273                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5274                     if !$conf->{keephugepages};
5275                 die $err;
5276             }
5277
5278             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5279                 if !$conf->{keephugepages};
5280         };
5281         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5282
5283     } else {
5284         eval { $run_qemu->() };
5285     }
5286
5287     if (my $err = $@) {
5288         # deactivate volumes if start fails
5289         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5290         die "start failed: $err";
5291     }
5292
5293     print "migration listens on $migrate_uri\n" if $migrate_uri;
5294     $res->{migrate_uri} = $migrate_uri;
5295
5296     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5297         eval { mon_cmd($vmid, "cont"); };
5298         warn $@ if $@;
5299     }
5300
5301     #start nbd server for storage migration
5302     if (my $nbd = $migrate_opts->{nbd}) {
5303         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5304
5305         my $migrate_storage_uri;
5306         # nbd_protocol_version > 0 for unix socket support
5307         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5308             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5309             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5310             $migrate_storage_uri = "nbd:unix:$socket_path";
5311         } else {
5312             my $nodename = nodename();
5313             my $localip = $get_migration_ip->($nodename);
5314             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5315             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5316
5317             mon_cmd($vmid, "nbd-server-start", addr => {
5318                 type => 'inet',
5319                 data => {
5320                     host => "${localip}",
5321                     port => "${storage_migrate_port}",
5322                 },
5323             });
5324             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5325             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5326         }
5327
5328         $res->{migrate_storage_uri} = $migrate_storage_uri;
5329
5330         foreach my $opt (sort keys %$nbd) {
5331             my $drivestr = $nbd->{$opt}->{drivestr};
5332             my $volid = $nbd->{$opt}->{volid};
5333             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5334             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5335             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5336             print "re-using replicated volume: $opt - $volid\n"
5337                 if $nbd->{$opt}->{replicated};
5338
5339             $res->{drives}->{$opt} = $nbd->{$opt};
5340             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5341         }
5342     }
5343
5344     if ($migratedfrom) {
5345         eval {
5346             set_migration_caps($vmid);
5347         };
5348         warn $@ if $@;
5349
5350         if ($spice_port) {
5351             print "spice listens on port $spice_port\n";
5352             $res->{spice_port} = $spice_port;
5353             if ($migrate_opts->{spice_ticket}) {
5354                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5355                     $migrate_opts->{spice_ticket});
5356                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5357             }
5358         }
5359
5360     } else {
5361         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5362             if !$statefile && $conf->{balloon};
5363
5364         foreach my $opt (keys %$conf) {
5365             next if $opt !~  m/^net\d+$/;
5366             my $nicconf = parse_net($conf->{$opt});
5367             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5368         }
5369     }
5370
5371     mon_cmd($vmid, 'qom-set',
5372                 path => "machine/peripheral/balloon0",
5373                 property => "guest-stats-polling-interval",
5374                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5375
5376     if ($resume) {
5377         print "Resumed VM, removing state\n";
5378         if (my $vmstate = $conf->{vmstate}) {
5379             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5380             PVE::Storage::vdisk_free($storecfg, $vmstate);
5381         }
5382         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5383         PVE::QemuConfig->write_config($vmid, $conf);
5384     }
5385
5386     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5387
5388     return $res;
5389 }
5390
5391 sub vm_commandline {
5392     my ($storecfg, $vmid, $snapname) = @_;
5393
5394     my $conf = PVE::QemuConfig->load_config($vmid);
5395     my $forcemachine;
5396     my $forcecpu;
5397
5398     if ($snapname) {
5399         my $snapshot = $conf->{snapshots}->{$snapname};
5400         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5401
5402         # check for machine or CPU overrides in snapshot
5403         $forcemachine = $snapshot->{runningmachine};
5404         $forcecpu = $snapshot->{runningcpu};
5405
5406         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5407
5408         $conf = $snapshot;
5409     }
5410
5411     my $defaults = load_defaults();
5412
5413     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5414         $forcemachine, $forcecpu);
5415
5416     return PVE::Tools::cmd2string($cmd);
5417 }
5418
5419 sub vm_reset {
5420     my ($vmid, $skiplock) = @_;
5421
5422     PVE::QemuConfig->lock_config($vmid, sub {
5423
5424         my $conf = PVE::QemuConfig->load_config($vmid);
5425
5426         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5427
5428         mon_cmd($vmid, "system_reset");
5429     });
5430 }
5431
5432 sub get_vm_volumes {
5433     my ($conf) = @_;
5434
5435     my $vollist = [];
5436     foreach_volid($conf, sub {
5437         my ($volid, $attr) = @_;
5438
5439         return if $volid =~ m|^/|;
5440
5441         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5442         return if !$sid;
5443
5444         push @$vollist, $volid;
5445     });
5446
5447     return $vollist;
5448 }
5449
5450 sub vm_stop_cleanup {
5451     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5452
5453     eval {
5454
5455         if (!$keepActive) {
5456             my $vollist = get_vm_volumes($conf);
5457             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5458         }
5459
5460         foreach my $ext (qw(mon qmp pid vnc qga)) {
5461             unlink "/var/run/qemu-server/${vmid}.$ext";
5462         }
5463
5464         if ($conf->{ivshmem}) {
5465             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5466             # just delete it for now, VMs which have this already open do not
5467             # are affected, but new VMs will get a separated one. If this
5468             # becomes an issue we either add some sort of ref-counting or just
5469             # add a "don't delete on stop" flag to the ivshmem format.
5470             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5471         }
5472
5473         foreach my $key (keys %$conf) {
5474             next if $key !~ m/^hostpci(\d+)$/;
5475             my $hostpciindex = $1;
5476             my $d = parse_hostpci($conf->{$key});
5477             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5478
5479             foreach my $pci (@{$d->{pciid}}) {
5480                 my $pciid = $pci->{id};
5481                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5482             }
5483         }
5484
5485         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5486     };
5487     warn $@ if $@; # avoid errors - just warn
5488 }
5489
5490 # call only in locked context
5491 sub _do_vm_stop {
5492     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5493
5494     my $pid = check_running($vmid, $nocheck);
5495     return if !$pid;
5496
5497     my $conf;
5498     if (!$nocheck) {
5499         $conf = PVE::QemuConfig->load_config($vmid);
5500         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5501         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5502             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5503             $timeout = $opts->{down} if $opts->{down};
5504         }
5505         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5506     }
5507
5508     eval {
5509         if ($shutdown) {
5510             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5511                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5512             } else {
5513                 mon_cmd($vmid, "system_powerdown");
5514             }
5515         } else {
5516             mon_cmd($vmid, "quit");
5517         }
5518     };
5519     my $err = $@;
5520
5521     if (!$err) {
5522         $timeout = 60 if !defined($timeout);
5523
5524         my $count = 0;
5525         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5526             $count++;
5527             sleep 1;
5528         }
5529
5530         if ($count >= $timeout) {
5531             if ($force) {
5532                 warn "VM still running - terminating now with SIGTERM\n";
5533                 kill 15, $pid;
5534             } else {
5535                 die "VM quit/powerdown failed - got timeout\n";
5536             }
5537         } else {
5538             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5539             return;
5540         }
5541     } else {
5542        if (!check_running($vmid, $nocheck)) {
5543             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5544             return;
5545        }
5546        if ($force) {
5547             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5548             kill 15, $pid;
5549         } else {
5550             die "VM quit/powerdown failed\n";
5551         }
5552     }
5553
5554     # wait again
5555     $timeout = 10;
5556
5557     my $count = 0;
5558     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5559         $count++;
5560         sleep 1;
5561     }
5562
5563     if ($count >= $timeout) {
5564         warn "VM still running - terminating now with SIGKILL\n";
5565         kill 9, $pid;
5566         sleep 1;
5567     }
5568
5569     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5570 }
5571
5572 # Note: use $nocheck to skip tests if VM configuration file exists.
5573 # We need that when migration VMs to other nodes (files already moved)
5574 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5575 sub vm_stop {
5576     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5577
5578     $force = 1 if !defined($force) && !$shutdown;
5579
5580     if ($migratedfrom){
5581         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5582         kill 15, $pid if $pid;
5583         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5584         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5585         return;
5586     }
5587
5588     PVE::QemuConfig->lock_config($vmid, sub {
5589         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5590    });
5591 }
5592
5593 sub vm_reboot {
5594     my ($vmid, $timeout) = @_;
5595
5596     PVE::QemuConfig->lock_config($vmid, sub {
5597         eval {
5598
5599             # only reboot if running, as qmeventd starts it again on a stop event
5600             return if !check_running($vmid);
5601
5602             create_reboot_request($vmid);
5603
5604             my $storecfg = PVE::Storage::config();
5605             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5606
5607         };
5608         if (my $err = $@) {
5609             # avoid that the next normal shutdown will be confused for a reboot
5610             clear_reboot_request($vmid);
5611             die $err;
5612         }
5613    });
5614 }
5615
5616 # note: if using the statestorage parameter, the caller has to check privileges
5617 sub vm_suspend {
5618     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5619
5620     my $conf;
5621     my $path;
5622     my $storecfg;
5623     my $vmstate;
5624
5625     PVE::QemuConfig->lock_config($vmid, sub {
5626
5627         $conf = PVE::QemuConfig->load_config($vmid);
5628
5629         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5630         PVE::QemuConfig->check_lock($conf)
5631             if !($skiplock || $is_backing_up);
5632
5633         die "cannot suspend to disk during backup\n"
5634             if $is_backing_up && $includestate;
5635
5636         if ($includestate) {
5637             $conf->{lock} = 'suspending';
5638             my $date = strftime("%Y-%m-%d", localtime(time()));
5639             $storecfg = PVE::Storage::config();
5640             if (!$statestorage) {
5641                 $statestorage = find_vmstate_storage($conf, $storecfg);
5642                 # check permissions for the storage
5643                 my $rpcenv = PVE::RPCEnvironment::get();
5644                 if ($rpcenv->{type} ne 'cli') {
5645                     my $authuser = $rpcenv->get_user();
5646                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5647                 }
5648             }
5649
5650
5651             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5652                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5653             $path = PVE::Storage::path($storecfg, $vmstate);
5654             PVE::QemuConfig->write_config($vmid, $conf);
5655         } else {
5656             mon_cmd($vmid, "stop");
5657         }
5658     });
5659
5660     if ($includestate) {
5661         # save vm state
5662         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5663
5664         eval {
5665             set_migration_caps($vmid, 1);
5666             mon_cmd($vmid, "savevm-start", statefile => $path);
5667             for(;;) {
5668                 my $state = mon_cmd($vmid, "query-savevm");
5669                 if (!$state->{status}) {
5670                     die "savevm not active\n";
5671                 } elsif ($state->{status} eq 'active') {
5672                     sleep(1);
5673                     next;
5674                 } elsif ($state->{status} eq 'completed') {
5675                     print "State saved, quitting\n";
5676                     last;
5677                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5678                     die "query-savevm failed with error '$state->{error}'\n"
5679                 } else {
5680                     die "query-savevm returned status '$state->{status}'\n";
5681                 }
5682             }
5683         };
5684         my $err = $@;
5685
5686         PVE::QemuConfig->lock_config($vmid, sub {
5687             $conf = PVE::QemuConfig->load_config($vmid);
5688             if ($err) {
5689                 # cleanup, but leave suspending lock, to indicate something went wrong
5690                 eval {
5691                     mon_cmd($vmid, "savevm-end");
5692                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5693                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5694                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5695                     PVE::QemuConfig->write_config($vmid, $conf);
5696                 };
5697                 warn $@ if $@;
5698                 die $err;
5699             }
5700
5701             die "lock changed unexpectedly\n"
5702                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5703
5704             mon_cmd($vmid, "quit");
5705             $conf->{lock} = 'suspended';
5706             PVE::QemuConfig->write_config($vmid, $conf);
5707         });
5708     }
5709 }
5710
5711 sub vm_resume {
5712     my ($vmid, $skiplock, $nocheck) = @_;
5713
5714     PVE::QemuConfig->lock_config($vmid, sub {
5715         my $res = mon_cmd($vmid, 'query-status');
5716         my $resume_cmd = 'cont';
5717         my $reset = 0;
5718
5719         if ($res->{status}) {
5720             return if $res->{status} eq 'running'; # job done, go home
5721             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5722             $reset = 1 if $res->{status} eq 'shutdown';
5723         }
5724
5725         if (!$nocheck) {
5726
5727             my $conf = PVE::QemuConfig->load_config($vmid);
5728
5729             PVE::QemuConfig->check_lock($conf)
5730                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5731         }
5732
5733         if ($reset) {
5734             # required if a VM shuts down during a backup and we get a resume
5735             # request before the backup finishes for example
5736             mon_cmd($vmid, "system_reset");
5737         }
5738         mon_cmd($vmid, $resume_cmd);
5739     });
5740 }
5741
5742 sub vm_sendkey {
5743     my ($vmid, $skiplock, $key) = @_;
5744
5745     PVE::QemuConfig->lock_config($vmid, sub {
5746
5747         my $conf = PVE::QemuConfig->load_config($vmid);
5748
5749         # there is no qmp command, so we use the human monitor command
5750         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5751         die $res if $res ne '';
5752     });
5753 }
5754
5755 # vzdump restore implementaion
5756
5757 sub tar_archive_read_firstfile {
5758     my $archive = shift;
5759
5760     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5761
5762     # try to detect archive type first
5763     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5764         die "unable to open file '$archive'\n";
5765     my $firstfile = <$fh>;
5766     kill 15, $pid;
5767     close $fh;
5768
5769     die "ERROR: archive contaions no data\n" if !$firstfile;
5770     chomp $firstfile;
5771
5772     return $firstfile;
5773 }
5774
5775 sub tar_restore_cleanup {
5776     my ($storecfg, $statfile) = @_;
5777
5778     print STDERR "starting cleanup\n";
5779
5780     if (my $fd = IO::File->new($statfile, "r")) {
5781         while (defined(my $line = <$fd>)) {
5782             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5783                 my $volid = $2;
5784                 eval {
5785                     if ($volid =~ m|^/|) {
5786                         unlink $volid || die 'unlink failed\n';
5787                     } else {
5788                         PVE::Storage::vdisk_free($storecfg, $volid);
5789                     }
5790                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5791                 };
5792                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5793             } else {
5794                 print STDERR "unable to parse line in statfile - $line";
5795             }
5796         }
5797         $fd->close();
5798     }
5799 }
5800
5801 sub restore_file_archive {
5802     my ($archive, $vmid, $user, $opts) = @_;
5803
5804     return restore_vma_archive($archive, $vmid, $user, $opts)
5805         if $archive eq '-';
5806
5807     my $info = PVE::Storage::archive_info($archive);
5808     my $format = $opts->{format} // $info->{format};
5809     my $comp = $info->{compression};
5810
5811     # try to detect archive format
5812     if ($format eq 'tar') {
5813         return restore_tar_archive($archive, $vmid, $user, $opts);
5814     } else {
5815         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5816     }
5817 }
5818
5819 # hepler to remove disks that will not be used after restore
5820 my $restore_cleanup_oldconf = sub {
5821     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5822
5823     PVE::QemuConfig->foreach_volume($oldconf, sub {
5824         my ($ds, $drive) = @_;
5825
5826         return if drive_is_cdrom($drive, 1);
5827
5828         my $volid = $drive->{file};
5829         return if !$volid || $volid =~ m|^/|;
5830
5831         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5832         return if !$path || !$owner || ($owner != $vmid);
5833
5834         # Note: only delete disk we want to restore
5835         # other volumes will become unused
5836         if ($virtdev_hash->{$ds}) {
5837             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5838             if (my $err = $@) {
5839                 warn $err;
5840             }
5841         }
5842     });
5843
5844     # delete vmstate files, after the restore we have no snapshots anymore
5845     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5846         my $snap = $oldconf->{snapshots}->{$snapname};
5847         if ($snap->{vmstate}) {
5848             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5849             if (my $err = $@) {
5850                 warn $err;
5851             }
5852         }
5853     }
5854 };
5855
5856 # Helper to parse vzdump backup device hints
5857 #
5858 # $rpcenv: Environment, used to ckeck storage permissions
5859 # $user: User ID, to check storage permissions
5860 # $storecfg: Storage configuration
5861 # $fh: the file handle for reading the configuration
5862 # $devinfo: should contain device sizes for all backu-up'ed devices
5863 # $options: backup options (pool, default storage)
5864 #
5865 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5866 my $parse_backup_hints = sub {
5867     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5868
5869     my $virtdev_hash = {};
5870
5871     while (defined(my $line = <$fh>)) {
5872         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5873             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5874             die "archive does not contain data for drive '$virtdev'\n"
5875                 if !$devinfo->{$devname};
5876
5877             if (defined($options->{storage})) {
5878                 $storeid = $options->{storage} || 'local';
5879             } elsif (!$storeid) {
5880                 $storeid = 'local';
5881             }
5882             $format = 'raw' if !$format;
5883             $devinfo->{$devname}->{devname} = $devname;
5884             $devinfo->{$devname}->{virtdev} = $virtdev;
5885             $devinfo->{$devname}->{format} = $format;
5886             $devinfo->{$devname}->{storeid} = $storeid;
5887
5888             # check permission on storage
5889             my $pool = $options->{pool}; # todo: do we need that?
5890             if ($user ne 'root@pam') {
5891                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5892             }
5893
5894             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5895         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5896             my $virtdev = $1;
5897             my $drive = parse_drive($virtdev, $2);
5898             if (drive_is_cloudinit($drive)) {
5899                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5900                 $storeid = $options->{storage} if defined ($options->{storage});
5901                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5902                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5903
5904                 $virtdev_hash->{$virtdev} = {
5905                     format => $format,
5906                     storeid => $storeid,
5907                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5908                     is_cloudinit => 1,
5909                 };
5910             }
5911         }
5912     }
5913
5914     return $virtdev_hash;
5915 };
5916
5917 # Helper to allocate and activate all volumes required for a restore
5918 #
5919 # $storecfg: Storage configuration
5920 # $virtdev_hash: as returned by parse_backup_hints()
5921 #
5922 # Returns: { $virtdev => $volid }
5923 my $restore_allocate_devices = sub {
5924     my ($storecfg, $virtdev_hash, $vmid) = @_;
5925
5926     my $map = {};
5927     foreach my $virtdev (sort keys %$virtdev_hash) {
5928         my $d = $virtdev_hash->{$virtdev};
5929         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5930         my $storeid = $d->{storeid};
5931         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5932
5933         # test if requested format is supported
5934         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5935         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5936         $d->{format} = $defFormat if !$supported;
5937
5938         my $name;
5939         if ($d->{is_cloudinit}) {
5940             $name = "vm-$vmid-cloudinit";
5941             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5942             if ($scfg->{path}) {
5943                 $name .= ".$d->{format}";
5944             }
5945         }
5946
5947         my $volid = PVE::Storage::vdisk_alloc(
5948             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5949
5950         print STDERR "new volume ID is '$volid'\n";
5951         $d->{volid} = $volid;
5952
5953         PVE::Storage::activate_volumes($storecfg, [$volid]);
5954
5955         $map->{$virtdev} = $volid;
5956     }
5957
5958     return $map;
5959 };
5960
5961 my $restore_update_config_line = sub {
5962     my ($cookie, $vmid, $map, $line, $unique) = @_;
5963
5964     return '' if $line =~ m/^\#qmdump\#/;
5965     return '' if $line =~ m/^\#vzdump\#/;
5966     return '' if $line =~ m/^lock:/;
5967     return '' if $line =~ m/^unused\d+:/;
5968     return '' if $line =~ m/^parent:/;
5969
5970     my $res = '';
5971
5972     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5973     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5974         # try to convert old 1.X settings
5975         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5976         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5977             my ($model, $macaddr) = split(/\=/, $devconfig);
5978             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5979             my $net = {
5980                 model => $model,
5981                 bridge => "vmbr$ind",
5982                 macaddr => $macaddr,
5983             };
5984             my $netstr = print_net($net);
5985
5986             $res .= "net$cookie->{netcount}: $netstr\n";
5987             $cookie->{netcount}++;
5988         }
5989     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5990         my ($id, $netstr) = ($1, $2);
5991         my $net = parse_net($netstr);
5992         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5993         $netstr = print_net($net);
5994         $res .= "$id: $netstr\n";
5995     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5996         my $virtdev = $1;
5997         my $value = $3;
5998         my $di = parse_drive($virtdev, $value);
5999         if (defined($di->{backup}) && !$di->{backup}) {
6000             $res .= "#$line";
6001         } elsif ($map->{$virtdev}) {
6002             delete $di->{format}; # format can change on restore
6003             $di->{file} = $map->{$virtdev};
6004             $value = print_drive($di);
6005             $res .= "$virtdev: $value\n";
6006         } else {
6007             $res .= $line;
6008         }
6009     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6010         my $vmgenid = $1;
6011         if ($vmgenid ne '0') {
6012             # always generate a new vmgenid if there was a valid one setup
6013             $vmgenid = generate_uuid();
6014         }
6015         $res .= "vmgenid: $vmgenid\n";
6016     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6017         my ($uuid, $uuid_str);
6018         UUID::generate($uuid);
6019         UUID::unparse($uuid, $uuid_str);
6020         my $smbios1 = parse_smbios1($2);
6021         $smbios1->{uuid} = $uuid_str;
6022         $res .= $1.print_smbios1($smbios1)."\n";
6023     } else {
6024         $res .= $line;
6025     }
6026
6027     return $res;
6028 };
6029
6030 my $restore_deactivate_volumes = sub {
6031     my ($storecfg, $devinfo) = @_;
6032
6033     my $vollist = [];
6034     foreach my $devname (keys %$devinfo) {
6035         my $volid = $devinfo->{$devname}->{volid};
6036         push @$vollist, $volid if $volid;
6037     }
6038
6039     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6040 };
6041
6042 my $restore_destroy_volumes = sub {
6043     my ($storecfg, $devinfo) = @_;
6044
6045     foreach my $devname (keys %$devinfo) {
6046         my $volid = $devinfo->{$devname}->{volid};
6047         next if !$volid;
6048         eval {
6049             if ($volid =~ m|^/|) {
6050                 unlink $volid || die 'unlink failed\n';
6051             } else {
6052                 PVE::Storage::vdisk_free($storecfg, $volid);
6053             }
6054             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6055         };
6056         print STDERR "unable to cleanup '$volid' - $@" if $@;
6057     }
6058 };
6059
6060 sub scan_volids {
6061     my ($cfg, $vmid) = @_;
6062
6063     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6064
6065     my $volid_hash = {};
6066     foreach my $storeid (keys %$info) {
6067         foreach my $item (@{$info->{$storeid}}) {
6068             next if !($item->{volid} && $item->{size});
6069             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6070             $volid_hash->{$item->{volid}} = $item;
6071         }
6072     }
6073
6074     return $volid_hash;
6075 }
6076
6077 sub update_disk_config {
6078     my ($vmid, $conf, $volid_hash) = @_;
6079
6080     my $changes;
6081     my $prefix = "VM $vmid";
6082
6083     # used and unused disks
6084     my $referenced = {};
6085
6086     # Note: it is allowed to define multiple storages with same path (alias), so
6087     # we need to check both 'volid' and real 'path' (two different volid can point
6088     # to the same path).
6089
6090     my $referencedpath = {};
6091
6092     # update size info
6093     PVE::QemuConfig->foreach_volume($conf, sub {
6094         my ($opt, $drive) = @_;
6095
6096         my $volid = $drive->{file};
6097         return if !$volid;
6098         my $volume = $volid_hash->{$volid};
6099
6100         # mark volid as "in-use" for next step
6101         $referenced->{$volid} = 1;
6102         if ($volume && (my $path = $volume->{path})) {
6103             $referencedpath->{$path} = 1;
6104         }
6105
6106         return if drive_is_cdrom($drive);
6107         return if !$volume;
6108
6109         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6110         if (defined($updated)) {
6111             $changes = 1;
6112             $conf->{$opt} = print_drive($updated);
6113             print "$prefix ($opt): $msg\n";
6114         }
6115     });
6116
6117     # remove 'unusedX' entry if volume is used
6118     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6119         my ($opt, $drive) = @_;
6120
6121         my $volid = $drive->{file};
6122         return if !$volid;
6123
6124         my $path;
6125         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6126         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6127             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6128             $changes = 1;
6129             delete $conf->{$opt};
6130         }
6131
6132         $referenced->{$volid} = 1;
6133         $referencedpath->{$path} = 1 if $path;
6134     });
6135
6136     foreach my $volid (sort keys %$volid_hash) {
6137         next if $volid =~ m/vm-$vmid-state-/;
6138         next if $referenced->{$volid};
6139         my $path = $volid_hash->{$volid}->{path};
6140         next if !$path; # just to be sure
6141         next if $referencedpath->{$path};
6142         $changes = 1;
6143         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6144         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6145         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6146     }
6147
6148     return $changes;
6149 }
6150
6151 sub rescan {
6152     my ($vmid, $nolock, $dryrun) = @_;
6153
6154     my $cfg = PVE::Storage::config();
6155
6156     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6157     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6158     foreach my $stor (keys %{$cfg->{ids}}) {
6159         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6160     }
6161
6162     print "rescan volumes...\n";
6163     my $volid_hash = scan_volids($cfg, $vmid);
6164
6165     my $updatefn =  sub {
6166         my ($vmid) = @_;
6167
6168         my $conf = PVE::QemuConfig->load_config($vmid);
6169
6170         PVE::QemuConfig->check_lock($conf);
6171
6172         my $vm_volids = {};
6173         foreach my $volid (keys %$volid_hash) {
6174             my $info = $volid_hash->{$volid};
6175             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6176         }
6177
6178         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6179
6180         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6181     };
6182
6183     if (defined($vmid)) {
6184         if ($nolock) {
6185             &$updatefn($vmid);
6186         } else {
6187             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6188         }
6189     } else {
6190         my $vmlist = config_list();
6191         foreach my $vmid (keys %$vmlist) {
6192             if ($nolock) {
6193                 &$updatefn($vmid);
6194             } else {
6195                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6196             }
6197         }
6198     }
6199 }
6200
6201 sub restore_proxmox_backup_archive {
6202     my ($archive, $vmid, $user, $options) = @_;
6203
6204     my $storecfg = PVE::Storage::config();
6205
6206     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6207     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6208
6209     my $fingerprint = $scfg->{fingerprint};
6210     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6211
6212     my $repo = PVE::PBSClient::get_repository($scfg);
6213
6214     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6215     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6216     local $ENV{PBS_PASSWORD} = $password;
6217     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6218
6219     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6220         PVE::Storage::parse_volname($storecfg, $archive);
6221
6222     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6223
6224     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6225
6226     my $tmpdir = "/var/tmp/vzdumptmp$$";
6227     rmtree $tmpdir;
6228     mkpath $tmpdir;
6229
6230     my $conffile = PVE::QemuConfig->config_file($vmid);
6231      # disable interrupts (always do cleanups)
6232     local $SIG{INT} =
6233         local $SIG{TERM} =
6234         local $SIG{QUIT} =
6235         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6236
6237     # Note: $oldconf is undef if VM does not exists
6238     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6239     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6240     my $new_conf_raw = '';
6241
6242     my $rpcenv = PVE::RPCEnvironment::get();
6243     my $devinfo = {};
6244
6245     eval {
6246         # enable interrupts
6247         local $SIG{INT} =
6248             local $SIG{TERM} =
6249             local $SIG{QUIT} =
6250             local $SIG{HUP} =
6251             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6252
6253         my $cfgfn = "$tmpdir/qemu-server.conf";
6254         my $firewall_config_fn = "$tmpdir/fw.conf";
6255         my $index_fn = "$tmpdir/index.json";
6256
6257         my $cmd = "restore";
6258
6259         my $param = [$pbs_backup_name, "index.json", $index_fn];
6260         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6261         my $index = PVE::Tools::file_get_contents($index_fn);
6262         $index = decode_json($index);
6263
6264         # print Dumper($index);
6265         foreach my $info (@{$index->{files}}) {
6266             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6267                 my $devname = $1;
6268                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6269                     $devinfo->{$devname}->{size} = $1;
6270                 } else {
6271                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6272                 }
6273             }
6274         }
6275
6276         my $is_qemu_server_backup = scalar(
6277             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6278         );
6279         if (!$is_qemu_server_backup) {
6280             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6281         }
6282         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6283
6284         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6285         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6286
6287         if ($has_firewall_config) {
6288             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6289             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6290
6291             my $pve_firewall_dir = '/etc/pve/firewall';
6292             mkdir $pve_firewall_dir; # make sure the dir exists
6293             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6294         }
6295
6296         my $fh = IO::File->new($cfgfn, "r") ||
6297             die "unable to read qemu-server.conf - $!\n";
6298
6299         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6300
6301         # fixme: rate limit?
6302
6303         # create empty/temp config
6304         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6305
6306         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6307
6308         # allocate volumes
6309         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6310
6311         if (!$options->{live}) {
6312             foreach my $virtdev (sort keys %$virtdev_hash) {
6313                 my $d = $virtdev_hash->{$virtdev};
6314                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6315
6316                 my $volid = $d->{volid};
6317
6318                 my $path = PVE::Storage::path($storecfg, $volid);
6319
6320                 my $pbs_restore_cmd = [
6321                     '/usr/bin/pbs-restore',
6322                     '--repository', $repo,
6323                     $pbs_backup_name,
6324                     "$d->{devname}.img.fidx",
6325                     $path,
6326                     '--verbose',
6327                     ];
6328
6329                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6330                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6331
6332                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6333                     push @$pbs_restore_cmd, '--skip-zero';
6334                 }
6335
6336                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6337                 print "restore proxmox backup image: $dbg_cmdstring\n";
6338                 run_command($pbs_restore_cmd);
6339             }
6340         }
6341
6342         $fh->seek(0, 0) || die "seek failed - $!\n";
6343
6344         my $cookie = { netcount => 0 };
6345         while (defined(my $line = <$fh>)) {
6346             $new_conf_raw .= $restore_update_config_line->(
6347                 $cookie,
6348                 $vmid,
6349                 $map,
6350                 $line,
6351                 $options->{unique},
6352             );
6353         }
6354
6355         $fh->close();
6356     };
6357     my $err = $@;
6358
6359     if ($err || !$options->{live}) {
6360         $restore_deactivate_volumes->($storecfg, $devinfo);
6361     }
6362
6363     rmtree $tmpdir;
6364
6365     if ($err) {
6366         $restore_destroy_volumes->($storecfg, $devinfo);
6367         die $err;
6368     }
6369
6370     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6371
6372     PVE::Cluster::cfs_update(); # make sure we read new file
6373
6374     eval { rescan($vmid, 1); };
6375     warn $@ if $@;
6376
6377     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6378
6379     if ($options->{live}) {
6380         eval {
6381             # enable interrupts
6382             local $SIG{INT} =
6383                 local $SIG{TERM} =
6384                 local $SIG{QUIT} =
6385                 local $SIG{HUP} =
6386                 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6387
6388             my $conf = PVE::QemuConfig->load_config($vmid);
6389             die "cannot do live-restore for template\n"
6390                 if PVE::QemuConfig->is_template($conf);
6391
6392             pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6393         };
6394
6395         $err = $@;
6396         if ($err) {
6397             warn "Detroying live-restore VM, all temporary data will be lost!\n";
6398             $restore_deactivate_volumes->($storecfg, $devinfo);
6399             $restore_destroy_volumes->($storecfg, $devinfo);
6400             unlink $conffile;
6401             die $err;
6402         }
6403     }
6404 }
6405
6406 sub pbs_live_restore {
6407     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6408
6409     print "Starting VM for live-restore\n";
6410
6411     my $pbs_backing = {};
6412     foreach my $ds (keys %$restored_disks) {
6413         $ds =~ m/^drive-(.*)$/;
6414         $pbs_backing->{$1} = {
6415             repository => $repo,
6416             snapshot => $snap,
6417             archive => "$ds.img.fidx",
6418         };
6419         $pbs_backing->{$1}->{keyfile} = $keyfile if -e $keyfile;
6420     }
6421
6422     eval {
6423         # make sure HA doesn't interrupt our restore by stopping the VM
6424         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6425             my $cmd = ['ha-manager', 'set',  "vm:$vmid", '--state', 'started'];
6426             PVE::Tools::run_command($cmd);
6427         }
6428
6429         # start VM with backing chain pointing to PBS backup, environment vars
6430         # for PBS driver in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already
6431         # set by our caller
6432         PVE::QemuServer::vm_start_nolock(
6433             $storecfg,
6434             $vmid,
6435             $conf,
6436             {
6437                 paused => 1,
6438                 'pbs-backing' => $pbs_backing,
6439             },
6440             {},
6441         );
6442
6443         my $qmeventd_fd = register_qmeventd_handle($vmid);
6444
6445         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6446         # this will effectively collapse the backing image chain consisting of
6447         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6448         # removes itself once all backing images vanish with 'auto-remove=on')
6449         my $jobs = {};
6450         foreach my $ds (keys %$restored_disks) {
6451             my $job_id = "restore-$ds";
6452             mon_cmd($vmid, 'block-stream',
6453                 'job-id' => $job_id,
6454                 device => "$ds",
6455             );
6456             $jobs->{$job_id} = {};
6457         }
6458
6459         mon_cmd($vmid, 'cont');
6460         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6461
6462         # all jobs finished, remove blockdevs now to disconnect from PBS
6463         foreach my $ds (keys %$restored_disks) {
6464             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6465         }
6466
6467         close($qmeventd_fd);
6468     };
6469
6470     my $err = $@;
6471
6472     if ($err) {
6473         warn "An error occured during live-restore: $err\n";
6474         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6475         die "live-restore failed\n";
6476     }
6477 }
6478
6479 sub restore_vma_archive {
6480     my ($archive, $vmid, $user, $opts, $comp) = @_;
6481
6482     my $readfrom = $archive;
6483
6484     my $cfg = PVE::Storage::config();
6485     my $commands = [];
6486     my $bwlimit = $opts->{bwlimit};
6487
6488     my $dbg_cmdstring = '';
6489     my $add_pipe = sub {
6490         my ($cmd) = @_;
6491         push @$commands, $cmd;
6492         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6493         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6494         $readfrom = '-';
6495     };
6496
6497     my $input = undef;
6498     if ($archive eq '-') {
6499         $input = '<&STDIN';
6500     } else {
6501         # If we use a backup from a PVE defined storage we also consider that
6502         # storage's rate limit:
6503         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6504         if (defined($volid)) {
6505             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6506             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6507             if ($readlimit) {
6508                 print STDERR "applying read rate limit: $readlimit\n";
6509                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6510                 $add_pipe->($cstream);
6511             }
6512         }
6513     }
6514
6515     if ($comp) {
6516         my $info = PVE::Storage::decompressor_info('vma', $comp);
6517         my $cmd = $info->{decompressor};
6518         push @$cmd, $readfrom;
6519         $add_pipe->($cmd);
6520     }
6521
6522     my $tmpdir = "/var/tmp/vzdumptmp$$";
6523     rmtree $tmpdir;
6524
6525     # disable interrupts (always do cleanups)
6526     local $SIG{INT} =
6527         local $SIG{TERM} =
6528         local $SIG{QUIT} =
6529         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6530
6531     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6532     POSIX::mkfifo($mapfifo, 0600);
6533     my $fifofh;
6534     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6535
6536     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6537
6538     my $oldtimeout;
6539     my $timeout = 5;
6540
6541     my $devinfo = {};
6542
6543     my $rpcenv = PVE::RPCEnvironment::get();
6544
6545     my $conffile = PVE::QemuConfig->config_file($vmid);
6546
6547     # Note: $oldconf is undef if VM does not exist
6548     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6549     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6550     my $new_conf_raw = '';
6551
6552     my %storage_limits;
6553
6554     my $print_devmap = sub {
6555         my $cfgfn = "$tmpdir/qemu-server.conf";
6556
6557         # we can read the config - that is already extracted
6558         my $fh = IO::File->new($cfgfn, "r") ||
6559             die "unable to read qemu-server.conf - $!\n";
6560
6561         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6562         if (-f $fwcfgfn) {
6563             my $pve_firewall_dir = '/etc/pve/firewall';
6564             mkdir $pve_firewall_dir; # make sure the dir exists
6565             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6566         }
6567
6568         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6569
6570         foreach my $info (values %{$virtdev_hash}) {
6571             my $storeid = $info->{storeid};
6572             next if defined($storage_limits{$storeid});
6573
6574             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6575             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6576             $storage_limits{$storeid} = $limit * 1024;
6577         }
6578
6579         foreach my $devname (keys %$devinfo) {
6580             die "found no device mapping information for device '$devname'\n"
6581                 if !$devinfo->{$devname}->{virtdev};
6582         }
6583
6584         # create empty/temp config
6585         if ($oldconf) {
6586             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6587             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6588         }
6589
6590         # allocate volumes
6591         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6592
6593         # print restore information to $fifofh
6594         foreach my $virtdev (sort keys %$virtdev_hash) {
6595             my $d = $virtdev_hash->{$virtdev};
6596             next if $d->{is_cloudinit}; # no need to restore cloudinit
6597
6598             my $storeid = $d->{storeid};
6599             my $volid = $d->{volid};
6600
6601             my $map_opts = '';
6602             if (my $limit = $storage_limits{$storeid}) {
6603                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6604             }
6605
6606             my $write_zeros = 1;
6607             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6608                 $write_zeros = 0;
6609             }
6610
6611             my $path = PVE::Storage::path($cfg, $volid);
6612
6613             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6614
6615             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6616         }
6617
6618         $fh->seek(0, 0) || die "seek failed - $!\n";
6619
6620         my $cookie = { netcount => 0 };
6621         while (defined(my $line = <$fh>)) {
6622             $new_conf_raw .= $restore_update_config_line->(
6623                 $cookie,
6624                 $vmid,
6625                 $map,
6626                 $line,
6627                 $opts->{unique},
6628             );
6629         }
6630
6631         $fh->close();
6632     };
6633
6634     eval {
6635         # enable interrupts
6636         local $SIG{INT} =
6637             local $SIG{TERM} =
6638             local $SIG{QUIT} =
6639             local $SIG{HUP} =
6640             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6641         local $SIG{ALRM} = sub { die "got timeout\n"; };
6642
6643         $oldtimeout = alarm($timeout);
6644
6645         my $parser = sub {
6646             my $line = shift;
6647
6648             print "$line\n";
6649
6650             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6651                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6652                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6653             } elsif ($line =~ m/^CTIME: /) {
6654                 # we correctly received the vma config, so we can disable
6655                 # the timeout now for disk allocation (set to 10 minutes, so
6656                 # that we always timeout if something goes wrong)
6657                 alarm(600);
6658                 &$print_devmap();
6659                 print $fifofh "done\n";
6660                 my $tmp = $oldtimeout || 0;
6661                 $oldtimeout = undef;
6662                 alarm($tmp);
6663                 close($fifofh);
6664                 $fifofh = undef;
6665             }
6666         };
6667
6668         print "restore vma archive: $dbg_cmdstring\n";
6669         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6670     };
6671     my $err = $@;
6672
6673     alarm($oldtimeout) if $oldtimeout;
6674
6675     $restore_deactivate_volumes->($cfg, $devinfo);
6676
6677     close($fifofh) if $fifofh;
6678     unlink $mapfifo;
6679     rmtree $tmpdir;
6680
6681     if ($err) {
6682         $restore_destroy_volumes->($cfg, $devinfo);
6683         die $err;
6684     }
6685
6686     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6687
6688     PVE::Cluster::cfs_update(); # make sure we read new file
6689
6690     eval { rescan($vmid, 1); };
6691     warn $@ if $@;
6692
6693     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6694 }
6695
6696 sub restore_tar_archive {
6697     my ($archive, $vmid, $user, $opts) = @_;
6698
6699     if ($archive ne '-') {
6700         my $firstfile = tar_archive_read_firstfile($archive);
6701         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6702             if $firstfile ne 'qemu-server.conf';
6703     }
6704
6705     my $storecfg = PVE::Storage::config();
6706
6707     # avoid zombie disks when restoring over an existing VM -> cleanup first
6708     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6709     # skiplock=1 because qmrestore has set the 'create' lock itself already
6710     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6711     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6712
6713     my $tocmd = "/usr/lib/qemu-server/qmextract";
6714
6715     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6716     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6717     $tocmd .= ' --prealloc' if $opts->{prealloc};
6718     $tocmd .= ' --info' if $opts->{info};
6719
6720     # tar option "xf" does not autodetect compression when read from STDIN,
6721     # so we pipe to zcat
6722     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6723         PVE::Tools::shellquote("--to-command=$tocmd");
6724
6725     my $tmpdir = "/var/tmp/vzdumptmp$$";
6726     mkpath $tmpdir;
6727
6728     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6729     local $ENV{VZDUMP_VMID} = $vmid;
6730     local $ENV{VZDUMP_USER} = $user;
6731
6732     my $conffile = PVE::QemuConfig->config_file($vmid);
6733     my $new_conf_raw = '';
6734
6735     # disable interrupts (always do cleanups)
6736     local $SIG{INT} =
6737         local $SIG{TERM} =
6738         local $SIG{QUIT} =
6739         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6740
6741     eval {
6742         # enable interrupts
6743         local $SIG{INT} =
6744             local $SIG{TERM} =
6745             local $SIG{QUIT} =
6746             local $SIG{HUP} =
6747             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6748
6749         if ($archive eq '-') {
6750             print "extracting archive from STDIN\n";
6751             run_command($cmd, input => "<&STDIN");
6752         } else {
6753             print "extracting archive '$archive'\n";
6754             run_command($cmd);
6755         }
6756
6757         return if $opts->{info};
6758
6759         # read new mapping
6760         my $map = {};
6761         my $statfile = "$tmpdir/qmrestore.stat";
6762         if (my $fd = IO::File->new($statfile, "r")) {
6763             while (defined (my $line = <$fd>)) {
6764                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6765                     $map->{$1} = $2 if $1;
6766                 } else {
6767                     print STDERR "unable to parse line in statfile - $line\n";
6768                 }
6769             }
6770             $fd->close();
6771         }
6772
6773         my $confsrc = "$tmpdir/qemu-server.conf";
6774
6775         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6776
6777         my $cookie = { netcount => 0 };
6778         while (defined (my $line = <$srcfd>)) {
6779             $new_conf_raw .= $restore_update_config_line->(
6780                 $cookie,
6781                 $vmid,
6782                 $map,
6783                 $line,
6784                 $opts->{unique},
6785             );
6786         }
6787
6788         $srcfd->close();
6789     };
6790     if (my $err = $@) {
6791         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6792         die $err;
6793     }
6794
6795     rmtree $tmpdir;
6796
6797     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6798
6799     PVE::Cluster::cfs_update(); # make sure we read new file
6800
6801     eval { rescan($vmid, 1); };
6802     warn $@ if $@;
6803 };
6804
6805 sub foreach_storage_used_by_vm {
6806     my ($conf, $func) = @_;
6807
6808     my $sidhash = {};
6809
6810     PVE::QemuConfig->foreach_volume($conf, sub {
6811         my ($ds, $drive) = @_;
6812         return if drive_is_cdrom($drive);
6813
6814         my $volid = $drive->{file};
6815
6816         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6817         $sidhash->{$sid} = $sid if $sid;
6818     });
6819
6820     foreach my $sid (sort keys %$sidhash) {
6821         &$func($sid);
6822     }
6823 }
6824
6825 my $qemu_snap_storage = {
6826     rbd => 1,
6827 };
6828 sub do_snapshots_with_qemu {
6829     my ($storecfg, $volid) = @_;
6830
6831     my $storage_name = PVE::Storage::parse_volume_id($volid);
6832     my $scfg = $storecfg->{ids}->{$storage_name};
6833     die "could not find storage '$storage_name'\n" if !defined($scfg);
6834
6835     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6836         return 1;
6837     }
6838
6839     if ($volid =~ m/\.(qcow2|qed)$/){
6840         return 1;
6841     }
6842
6843     return;
6844 }
6845
6846 sub qga_check_running {
6847     my ($vmid, $nowarn) = @_;
6848
6849     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6850     if ($@) {
6851         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6852         return 0;
6853     }
6854     return 1;
6855 }
6856
6857 sub template_create {
6858     my ($vmid, $conf, $disk) = @_;
6859
6860     my $storecfg = PVE::Storage::config();
6861
6862     PVE::QemuConfig->foreach_volume($conf, sub {
6863         my ($ds, $drive) = @_;
6864
6865         return if drive_is_cdrom($drive);
6866         return if $disk && $ds ne $disk;
6867
6868         my $volid = $drive->{file};
6869         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6870
6871         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6872         $drive->{file} = $voliddst;
6873         $conf->{$ds} = print_drive($drive);
6874         PVE::QemuConfig->write_config($vmid, $conf);
6875     });
6876 }
6877
6878 sub convert_iscsi_path {
6879     my ($path) = @_;
6880
6881     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6882         my $portal = $1;
6883         my $target = $2;
6884         my $lun = $3;
6885
6886         my $initiator_name = get_initiator_name();
6887
6888         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6889                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6890     }
6891
6892     die "cannot convert iscsi path '$path', unkown format\n";
6893 }
6894
6895 sub qemu_img_convert {
6896     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6897
6898     my $storecfg = PVE::Storage::config();
6899     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6900     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6901
6902     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6903
6904     my $cachemode;
6905     my $src_path;
6906     my $src_is_iscsi = 0;
6907     my $src_format;
6908
6909     if ($src_storeid) {
6910         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6911         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6912         $src_format = qemu_img_format($src_scfg, $src_volname);
6913         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6914         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6915         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6916     } elsif (-f $src_volid) {
6917         $src_path = $src_volid;
6918         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6919             $src_format = $1;
6920         }
6921     }
6922
6923     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6924
6925     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6926     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6927     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6928     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6929
6930     my $cmd = [];
6931     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6932     push @$cmd, '-l', "snapshot.name=$snapname"
6933         if $snapname && $src_format && $src_format eq "qcow2";
6934     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6935     push @$cmd, '-T', $cachemode if defined($cachemode);
6936
6937     if ($src_is_iscsi) {
6938         push @$cmd, '--image-opts';
6939         $src_path = convert_iscsi_path($src_path);
6940     } elsif ($src_format) {
6941         push @$cmd, '-f', $src_format;
6942     }
6943
6944     if ($dst_is_iscsi) {
6945         push @$cmd, '--target-image-opts';
6946         $dst_path = convert_iscsi_path($dst_path);
6947     } else {
6948         push @$cmd, '-O', $dst_format;
6949     }
6950
6951     push @$cmd, $src_path;
6952
6953     if (!$dst_is_iscsi && $is_zero_initialized) {
6954         push @$cmd, "zeroinit:$dst_path";
6955     } else {
6956         push @$cmd, $dst_path;
6957     }
6958
6959     my $parser = sub {
6960         my $line = shift;
6961         if($line =~ m/\((\S+)\/100\%\)/){
6962             my $percent = $1;
6963             my $transferred = int($size * $percent / 100);
6964             my $remaining = $size - $transferred;
6965
6966             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6967         }
6968
6969     };
6970
6971     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6972     my $err = $@;
6973     die "copy failed: $err" if $err;
6974 }
6975
6976 sub qemu_img_format {
6977     my ($scfg, $volname) = @_;
6978
6979     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6980         return $1;
6981     } else {
6982         return "raw";
6983     }
6984 }
6985
6986 sub qemu_drive_mirror {
6987     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6988
6989     $jobs = {} if !$jobs;
6990
6991     my $qemu_target;
6992     my $format;
6993     $jobs->{"drive-$drive"} = {};
6994
6995     if ($dst_volid =~ /^nbd:/) {
6996         $qemu_target = $dst_volid;
6997         $format = "nbd";
6998     } else {
6999         my $storecfg = PVE::Storage::config();
7000         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7001
7002         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7003
7004         $format = qemu_img_format($dst_scfg, $dst_volname);
7005
7006         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7007
7008         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7009     }
7010
7011     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7012     $opts->{format} = $format if $format;
7013
7014     if (defined($src_bitmap)) {
7015         $opts->{sync} = 'incremental';
7016         $opts->{bitmap} = $src_bitmap;
7017         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7018     }
7019
7020     if (defined($bwlimit)) {
7021         $opts->{speed} = $bwlimit * 1024;
7022         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7023     } else {
7024         print "drive mirror is starting for drive-$drive\n";
7025     }
7026
7027     # if a job already runs for this device we get an error, catch it for cleanup
7028     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7029     if (my $err = $@) {
7030         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7031         warn "$@\n" if $@;
7032         die "mirroring error: $err\n";
7033     }
7034
7035     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7036 }
7037
7038 # $completion can be either
7039 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7040 # 'cancel': wait until all jobs are ready, block-job-cancel them
7041 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7042 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7043 sub qemu_drive_mirror_monitor {
7044     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7045
7046     $completion //= 'complete';
7047     $op //= "mirror";
7048
7049     eval {
7050         my $err_complete = 0;
7051
7052         while (1) {
7053             die "block job ('$op') timed out\n" if $err_complete > 300;
7054
7055             my $stats = mon_cmd($vmid, "query-block-jobs");
7056
7057             my $running_jobs = {};
7058             foreach my $stat (@$stats) {
7059                 next if $stat->{type} ne $op;
7060                 $running_jobs->{$stat->{device}} = $stat;
7061             }
7062
7063             my $readycounter = 0;
7064
7065             foreach my $job (keys %$jobs) {
7066
7067                 my $vanished = !defined($running_jobs->{$job});
7068                 my $complete = defined($jobs->{$job}->{complete}) && $vanished;
7069                 if($complete || ($vanished && $completion eq 'auto')) {
7070                     print "$job: finished\n";
7071                     delete $jobs->{$job};
7072                     next;
7073                 }
7074
7075                 die "$job: '$op' has been cancelled\n" if !defined($running_jobs->{$job});
7076
7077                 my $busy = $running_jobs->{$job}->{busy};
7078                 my $ready = $running_jobs->{$job}->{ready};
7079                 if (my $total = $running_jobs->{$job}->{len}) {
7080                     my $transferred = $running_jobs->{$job}->{offset} || 0;
7081                     my $remaining = $total - $transferred;
7082                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7083
7084                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7085                 }
7086
7087                 $readycounter++ if $running_jobs->{$job}->{ready};
7088             }
7089
7090             last if scalar(keys %$jobs) == 0;
7091
7092             if ($readycounter == scalar(keys %$jobs)) {
7093                 print "all '$op' jobs are ready\n";
7094
7095                 # do the complete later (or has already been done)
7096                 last if $completion eq 'skip' || $completion eq 'auto';
7097
7098                 if ($vmiddst && $vmiddst != $vmid) {
7099                     my $agent_running = $qga && qga_check_running($vmid);
7100                     if ($agent_running) {
7101                         print "freeze filesystem\n";
7102                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7103                     } else {
7104                         print "suspend vm\n";
7105                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7106                     }
7107
7108                     # if we clone a disk for a new target vm, we don't switch the disk
7109                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7110
7111                     if ($agent_running) {
7112                         print "unfreeze filesystem\n";
7113                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7114                     } else {
7115                         print "resume vm\n";
7116                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7117                     }
7118
7119                     last;
7120                 } else {
7121
7122                     foreach my $job (keys %$jobs) {
7123                         # try to switch the disk if source and destination are on the same guest
7124                         print "$job: Completing block job...\n";
7125
7126                         my $op;
7127                         if ($completion eq 'complete') {
7128                             $op = 'block-job-complete';
7129                         } elsif ($completion eq 'cancel') {
7130                             $op = 'block-job-cancel';
7131                         } else {
7132                             die "invalid completion value: $completion\n";
7133                         }
7134                         eval { mon_cmd($vmid, $op, device => $job) };
7135                         if ($@ =~ m/cannot be completed/) {
7136                             print "$job: Block job cannot be completed, try again.\n";
7137                             $err_complete++;
7138                         }else {
7139                             print "$job: Completed successfully.\n";
7140                             $jobs->{$job}->{complete} = 1;
7141                         }
7142                     }
7143                 }
7144             }
7145             sleep 1;
7146         }
7147     };
7148     my $err = $@;
7149
7150     if ($err) {
7151         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7152         die "block job ('$op') error: $err";
7153     }
7154
7155 }
7156
7157 sub qemu_blockjobs_cancel {
7158     my ($vmid, $jobs) = @_;
7159
7160     foreach my $job (keys %$jobs) {
7161         print "$job: Cancelling block job\n";
7162         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7163         $jobs->{$job}->{cancel} = 1;
7164     }
7165
7166     while (1) {
7167         my $stats = mon_cmd($vmid, "query-block-jobs");
7168
7169         my $running_jobs = {};
7170         foreach my $stat (@$stats) {
7171             $running_jobs->{$stat->{device}} = $stat;
7172         }
7173
7174         foreach my $job (keys %$jobs) {
7175
7176             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7177                 print "$job: Done.\n";
7178                 delete $jobs->{$job};
7179             }
7180         }
7181
7182         last if scalar(keys %$jobs) == 0;
7183
7184         sleep 1;
7185     }
7186 }
7187
7188 sub clone_disk {
7189     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7190         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7191
7192     my $newvolid;
7193
7194     if (!$full) {
7195         print "create linked clone of drive $drivename ($drive->{file})\n";
7196         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7197         push @$newvollist, $newvolid;
7198     } else {
7199
7200         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7201         $storeid = $storage if $storage;
7202
7203         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7204
7205         print "create full clone of drive $drivename ($drive->{file})\n";
7206         my $name = undef;
7207         my $size = undef;
7208         if (drive_is_cloudinit($drive)) {
7209             $name = "vm-$newvmid-cloudinit";
7210             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7211             if ($scfg->{path}) {
7212                 $name .= ".$dst_format";
7213             }
7214             $snapname = undef;
7215             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7216         } elsif ($drivename eq 'efidisk0') {
7217             $size = get_efivars_size($conf);
7218         } else {
7219             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7220         }
7221         $newvolid = PVE::Storage::vdisk_alloc(
7222             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7223         );
7224         push @$newvollist, $newvolid;
7225
7226         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7227
7228         if (drive_is_cloudinit($drive)) {
7229             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7230             # if this is the case, we have to complete any block-jobs still there from
7231             # previous drive-mirrors
7232             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7233                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7234             }
7235             goto no_data_clone;
7236         }
7237
7238         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7239         if (!$running || $snapname) {
7240             # TODO: handle bwlimits
7241             if ($drivename eq 'efidisk0') {
7242                 # the relevant data on the efidisk may be smaller than the source
7243                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7244                 # that is given by the OVMF_VARS.fd
7245                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7246                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7247
7248                 # better for Ceph if block size is not too small, see bug #3324
7249                 my $bs = 1024*1024;
7250
7251                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7252                     "if=$src_path", "of=$dst_path"]);
7253             } else {
7254                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7255             }
7256         } else {
7257
7258             my $kvmver = get_running_qemu_version ($vmid);
7259             if (!min_version($kvmver, 2, 7)) {
7260                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7261                     if $drive->{iothread};
7262             }
7263
7264             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7265                 $completion, $qga, $bwlimit);
7266         }
7267     }
7268
7269 no_data_clone:
7270     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7271
7272     my $disk = $drive;
7273     $disk->{format} = undef;
7274     $disk->{file} = $newvolid;
7275     $disk->{size} = $size if defined($size);
7276
7277     return $disk;
7278 }
7279
7280 sub get_running_qemu_version {
7281     my ($vmid) = @_;
7282     my $res = mon_cmd($vmid, "query-version");
7283     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7284 }
7285
7286 sub qemu_use_old_bios_files {
7287     my ($machine_type) = @_;
7288
7289     return if !$machine_type;
7290
7291     my $use_old_bios_files = undef;
7292
7293     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7294         $machine_type = $1;
7295         $use_old_bios_files = 1;
7296     } else {
7297         my $version = extract_version($machine_type, kvm_user_version());
7298         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7299         # load new efi bios files on migration. So this hack is required to allow
7300         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7301         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7302         $use_old_bios_files = !min_version($version, 2, 4);
7303     }
7304
7305     return ($use_old_bios_files, $machine_type);
7306 }
7307
7308 sub get_efivars_size {
7309     my ($conf) = @_;
7310     my $arch = get_vm_arch($conf);
7311     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7312     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7313     return -s $ovmf_vars;
7314 }
7315
7316 sub update_efidisk_size {
7317     my ($conf) = @_;
7318
7319     return if !defined($conf->{efidisk0});
7320
7321     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7322     $disk->{size} = get_efivars_size($conf);
7323     $conf->{efidisk0} = print_drive($disk);
7324
7325     return;
7326 }
7327
7328 sub create_efidisk($$$$$) {
7329     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7330
7331     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7332     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7333
7334     my $vars_size_b = -s $ovmf_vars;
7335     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7336     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7337     PVE::Storage::activate_volumes($storecfg, [$volid]);
7338
7339     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7340     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7341
7342     return ($volid, $size/1024);
7343 }
7344
7345 sub vm_iothreads_list {
7346     my ($vmid) = @_;
7347
7348     my $res = mon_cmd($vmid, 'query-iothreads');
7349
7350     my $iothreads = {};
7351     foreach my $iothread (@$res) {
7352         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7353     }
7354
7355     return $iothreads;
7356 }
7357
7358 sub scsihw_infos {
7359     my ($conf, $drive) = @_;
7360
7361     my $maxdev = 0;
7362
7363     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7364         $maxdev = 7;
7365     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7366         $maxdev = 1;
7367     } else {
7368         $maxdev = 256;
7369     }
7370
7371     my $controller = int($drive->{index} / $maxdev);
7372     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7373         ? "virtioscsi"
7374         : "scsihw";
7375
7376     return ($maxdev, $controller, $controller_prefix);
7377 }
7378
7379 sub windows_version {
7380     my ($ostype) = @_;
7381
7382     return 0 if !$ostype;
7383
7384     my $winversion = 0;
7385
7386     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7387         $winversion = 5;
7388     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7389         $winversion = 6;
7390     } elsif ($ostype =~ m/^win(\d+)$/) {
7391         $winversion = $1;
7392     }
7393
7394     return $winversion;
7395 }
7396
7397 sub resolve_dst_disk_format {
7398         my ($storecfg, $storeid, $src_volname, $format) = @_;
7399         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7400
7401         if (!$format) {
7402             # if no target format is specified, use the source disk format as hint
7403             if ($src_volname) {
7404                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7405                 $format = qemu_img_format($scfg, $src_volname);
7406             } else {
7407                 return $defFormat;
7408             }
7409         }
7410
7411         # test if requested format is supported - else use default
7412         my $supported = grep { $_ eq $format } @$validFormats;
7413         $format = $defFormat if !$supported;
7414         return $format;
7415 }
7416
7417 # NOTE: if this logic changes, please update docs & possibly gui logic
7418 sub find_vmstate_storage {
7419     my ($conf, $storecfg) = @_;
7420
7421     # first, return storage from conf if set
7422     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7423
7424     my ($target, $shared, $local);
7425
7426     foreach_storage_used_by_vm($conf, sub {
7427         my ($sid) = @_;
7428         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7429         my $dst = $scfg->{shared} ? \$shared : \$local;
7430         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7431     });
7432
7433     # second, use shared storage where VM has at least one disk
7434     # third, use local storage where VM has at least one disk
7435     # fall back to local storage
7436     $target = $shared // $local // 'local';
7437
7438     return $target;
7439 }
7440
7441 sub generate_uuid {
7442     my ($uuid, $uuid_str);
7443     UUID::generate($uuid);
7444     UUID::unparse($uuid, $uuid_str);
7445     return $uuid_str;
7446 }
7447
7448 sub generate_smbios1_uuid {
7449     return "uuid=".generate_uuid();
7450 }
7451
7452 sub nbd_stop {
7453     my ($vmid) = @_;
7454
7455     mon_cmd($vmid, 'nbd-server-stop');
7456 }
7457
7458 sub create_reboot_request {
7459     my ($vmid) = @_;
7460     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7461         or die "failed to create reboot trigger file: $!\n";
7462     close($fh);
7463 }
7464
7465 sub clear_reboot_request {
7466     my ($vmid) = @_;
7467     my $path = "/run/qemu-server/$vmid.reboot";
7468     my $res = 0;
7469
7470     $res = unlink($path);
7471     die "could not remove reboot request for $vmid: $!"
7472         if !$res && $! != POSIX::ENOENT;
7473
7474     return $res;
7475 }
7476
7477 sub bootorder_from_legacy {
7478     my ($conf, $bootcfg) = @_;
7479
7480     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7481     my $bootindex_hash = {};
7482     my $i = 1;
7483     foreach my $o (split(//, $boot)) {
7484         $bootindex_hash->{$o} = $i*100;
7485         $i++;
7486     }
7487
7488     my $bootorder = {};
7489
7490     PVE::QemuConfig->foreach_volume($conf, sub {
7491         my ($ds, $drive) = @_;
7492
7493         if (drive_is_cdrom ($drive, 1)) {
7494             if ($bootindex_hash->{d}) {
7495                 $bootorder->{$ds} = $bootindex_hash->{d};
7496                 $bootindex_hash->{d} += 1;
7497             }
7498         } elsif ($bootindex_hash->{c}) {
7499             $bootorder->{$ds} = $bootindex_hash->{c}
7500                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7501             $bootindex_hash->{c} += 1;
7502         }
7503     });
7504
7505     if ($bootindex_hash->{n}) {
7506         for (my $i = 0; $i < $MAX_NETS; $i++) {
7507             my $netname = "net$i";
7508             next if !$conf->{$netname};
7509             $bootorder->{$netname} = $bootindex_hash->{n};
7510             $bootindex_hash->{n} += 1;
7511         }
7512     }
7513
7514     return $bootorder;
7515 }
7516
7517 # Generate default device list for 'boot: order=' property. Matches legacy
7518 # default boot order, but with explicit device names. This is important, since
7519 # the fallback for when neither 'order' nor the old format is specified relies
7520 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7521 sub get_default_bootdevices {
7522     my ($conf) = @_;
7523
7524     my @ret = ();
7525
7526     # harddisk
7527     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7528     push @ret, $first if $first;
7529
7530     # cdrom
7531     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7532     push @ret, $first if $first;
7533
7534     # network
7535     for (my $i = 0; $i < $MAX_NETS; $i++) {
7536         my $netname = "net$i";
7537         next if !$conf->{$netname};
7538         push @ret, $netname;
7539         last;
7540     }
7541
7542     return \@ret;
7543 }
7544
7545 sub device_bootorder {
7546     my ($conf) = @_;
7547
7548     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7549
7550     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7551
7552     my $bootorder = {};
7553     if (!defined($boot) || $boot->{legacy}) {
7554         $bootorder = bootorder_from_legacy($conf, $boot);
7555     } elsif ($boot->{order}) {
7556         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7557         for my $dev (PVE::Tools::split_list($boot->{order})) {
7558             $bootorder->{$dev} = $i++;
7559         }
7560     }
7561
7562     return $bootorder;
7563 }
7564
7565 sub register_qmeventd_handle {
7566     my ($vmid) = @_;
7567
7568     my $fh;
7569     my $peer = "/var/run/qmeventd.sock";
7570     my $count = 0;
7571
7572     for (;;) {
7573         $count++;
7574         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7575         last if $fh;
7576         if ($! != EINTR && $! != EAGAIN) {
7577             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7578         }
7579         if ($count > 4) {
7580             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7581               . "after $count retries\n";
7582         }
7583         usleep(25000);
7584     }
7585
7586     # send handshake to mark VM as backing up
7587     print $fh to_json({vzdump => {vmid => "$vmid"}});
7588
7589     # return handle to be closed later when inhibit is no longer required
7590     return $fh;
7591 }
7592
7593 # bash completion helper
7594
7595 sub complete_backup_archives {
7596     my ($cmdname, $pname, $cvalue) = @_;
7597
7598     my $cfg = PVE::Storage::config();
7599
7600     my $storeid;
7601
7602     if ($cvalue =~ m/^([^:]+):/) {
7603         $storeid = $1;
7604     }
7605
7606     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7607
7608     my $res = [];
7609     foreach my $id (keys %$data) {
7610         foreach my $item (@{$data->{$id}}) {
7611             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7612             push @$res, $item->{volid} if defined($item->{volid});
7613         }
7614     }
7615
7616     return $res;
7617 }
7618
7619 my $complete_vmid_full = sub {
7620     my ($running) = @_;
7621
7622     my $idlist = vmstatus();
7623
7624     my $res = [];
7625
7626     foreach my $id (keys %$idlist) {
7627         my $d = $idlist->{$id};
7628         if (defined($running)) {
7629             next if $d->{template};
7630             next if $running && $d->{status} ne 'running';
7631             next if !$running && $d->{status} eq 'running';
7632         }
7633         push @$res, $id;
7634
7635     }
7636     return $res;
7637 };
7638
7639 sub complete_vmid {
7640     return &$complete_vmid_full();
7641 }
7642
7643 sub complete_vmid_stopped {
7644     return &$complete_vmid_full(0);
7645 }
7646
7647 sub complete_vmid_running {
7648     return &$complete_vmid_full(1);
7649 }
7650
7651 sub complete_storage {
7652
7653     my $cfg = PVE::Storage::config();
7654     my $ids = $cfg->{ids};
7655
7656     my $res = [];
7657     foreach my $sid (keys %$ids) {
7658         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7659         next if !$ids->{$sid}->{content}->{images};
7660         push @$res, $sid;
7661     }
7662
7663     return $res;
7664 }
7665
7666 sub complete_migration_storage {
7667     my ($cmd, $param, $current_value, $all_args) = @_;
7668
7669     my $targetnode = @$all_args[1];
7670
7671     my $cfg = PVE::Storage::config();
7672     my $ids = $cfg->{ids};
7673
7674     my $res = [];
7675     foreach my $sid (keys %$ids) {
7676         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7677         next if !$ids->{$sid}->{content}->{images};
7678         push @$res, $sid;
7679     }
7680
7681     return $res;
7682 }
7683
7684 sub vm_is_paused {
7685     my ($vmid) = @_;
7686     my $qmpstatus = eval {
7687         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7688         mon_cmd($vmid, "query-status");
7689     };
7690     warn "$@\n" if $@;
7691     return $qmpstatus && $qmpstatus->{status} eq "paused";
7692 }
7693
7694 1;